{"level":"info","ts":"2026-06-10T10:17:11.703Z","logger":"setup","msg":"Loaded feature gates","featureGates":{"RayClusterStatusConditions":true,"RayJobDeletionPolicy":false}}
{"level":"info","ts":"2026-06-10T10:17:11.703Z","logger":"setup","msg":"Flag watchNamespace is not set. Watch custom resources in all namespaces."}
{"level":"info","ts":"2026-06-10T10:17:11.703Z","logger":"setup","msg":"Setup manager"}
{"level":"info","ts":"2026-06-10T10:17:11.902Z","logger":"setup","msg":"NetworkPolicy controller registered (annotation-based activation)"}
{"level":"info","ts":"2026-06-10T10:17:11.902Z","logger":"controller-runtime.builder","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-10T10:17:11.902Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-10T10:17:11.902Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-10T10:17:11.902Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-06-10T10:17:11.903Z","logger":"setup","msg":"starting manager"}
{"level":"info","ts":"2026-06-10T10:17:11.903Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-06-10T10:17:11.903Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8080","secure":false}
{"level":"info","ts":"2026-06-10T10:17:11.903Z","msg":"starting server","name":"health probe","addr":"[::]:8082"}
{"level":"info","ts":"2026-06-10T10:17:11.903Z","logger":"controller-runtime.webhook","msg":"Starting webhook server"}
{"level":"info","ts":"2026-06-10T10:17:11.903Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-06-10T10:17:11.903Z","logger":"controller-runtime.webhook","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-06-10T10:17:11.920Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"info","ts":"2026-06-10T10:17:12.120Z","msg":"attempting to acquire leader lease opendatahub/ray-operator-leader..."}
{"level":"info","ts":"2026-06-10T10:17:12.127Z","msg":"successfully acquired lease opendatahub/ray-operator-leader"}
{"level":"info","ts":"2026-06-10T10:17:12.127Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-10T10:17:12.127Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-10T10:17:12.127Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Job"}
{"level":"info","ts":"2026-06-10T10:17:12.127Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayJob"}
{"level":"info","ts":"2026-06-10T10:17:12.127Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-10T10:17:12.127Z","msg":"Starting EventSource","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.NetworkPolicy"}
{"level":"info","ts":"2026-06-10T10:17:12.127Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-06-10T10:17:12.127Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-10T10:17:12.128Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-10T10:17:12.128Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-10T10:17:12.129Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-06-10T10:17:12.129Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-10T10:17:12.129Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Route"}
{"level":"info","ts":"2026-06-10T10:17:12.129Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-06-10T10:17:12.129Z","msg":"Starting EventSource","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-10T10:17:12.129Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-06-10T10:17:12.128Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayService"}
{"level":"info","ts":"2026-06-10T10:17:12.600Z","msg":"Starting Controller","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-06-10T10:17:12.601Z","msg":"Starting workers","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","logger":"controllers.RayService","msg":"Starting Controller"}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","msg":"Starting Controller","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","logger":"controllers.RayService","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","msg":"Starting workers","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","logger":"controllers.RayCluster","msg":"Starting Controller"}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","logger":"controllers.RayJob","msg":"Starting Controller"}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","logger":"controllers.RayCluster","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","logger":"controllers.RayJob","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","msg":"Starting Controller","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-06-10T10:17:12.699Z","msg":"Starting workers","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-06-10T10:19:51.771Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.771Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-qqsjt"}
{"level":"info","ts":"2026-06-10T10:19:51.794Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"1a194d5d-1773-41f3-b5fc-923d2c6f00e6","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.794Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"}}
{"level":"info","ts":"2026-06-10T10:19:51.794Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589"}
{"level":"info","ts":"2026-06-10T10:19:51.794Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589"}
{"level":"info","ts":"2026-06-10T10:19:51.794Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.794Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.795Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.794Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.801Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.801Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-qqsjt"}
{"level":"info","ts":"2026-06-10T10:19:51.821Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","name":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:51.828Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"1a194d5d-1773-41f3-b5fc-923d2c6f00e6","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.828Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"37c70120-1965-4f81-b58f-84989afb5a33","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.838Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","name":"raycluster-auth-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:19:51.899Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:51.899Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"37c70120-1965-4f81-b58f-84989afb5a33","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:19:51.999Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:52.002Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"37c70120-1965-4f81-b58f-84989afb5a33","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:19:52.003Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"37c70120-1965-4f81-b58f-84989afb5a33","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:52.099Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","name":"kuberay-gateway-access","namespace":"test-ns-qqsjt","operation":"created"}
{"level":"info","ts":"2026-06-10T10:19:52.099Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:52.199Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14"}
{"level":"info","ts":"2026-06-10T10:19:52.199Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14","rayCluster":"raycluster-auth","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:19:52.199Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:52.207Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","name":"test-ns-qqsjt-raycluster-auth","operation":"created"}
{"level":"info","ts":"2026-06-10T10:19:52.299Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:52.310Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14"}
{"level":"info","ts":"2026-06-10T10:19:52.310Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:19:52.399Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14"}
{"level":"info","ts":"2026-06-10T10:19:52.399Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:19:52.399Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"100727bc-483b-4664-afe0-d8d976a9ba14"}
{"level":"info","ts":"2026-06-10T10:19:52.399Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"216d1be8-945b-4cf4-b2b8-12e8f0181320","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:52.399Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"216d1be8-945b-4cf4-b2b8-12e8f0181320"}
{"level":"info","ts":"2026-06-10T10:19:52.922Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-10T10:19:52.923Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:52.923Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-10T10:19:52.999Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:52.999Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:52.999Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:52.999Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:19:52.999Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:52Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:52Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:52Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:52Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-qqsjt","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589"}
{"level":"error","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"a9e20e66-e909-4d6d-9e47-d23f11cf8589","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39"}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39"}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:19:53.007Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39"}
{"level":"error","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"78668899-734e-4fe2-b9db-f4f11be1cf39","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1"}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1"}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","name":"raycluster-auth","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"d2b2cffb-2c27-4160-9b06-eed7de59e1b1","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"913da36b-5995-4924-a0b9-402fbeec4e5d","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.017Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"913da36b-5995-4924-a0b9-402fbeec4e5d"}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-qqsjt","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1"}
{"level":"error","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d4ffcb2a-3f6e-47f0-b41a-fa7c65df1fe1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9"}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9"}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9"}
{"level":"error","ts":"2026-06-10T10:19:53.099Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e4ac05dd-2932-4123-a78e-93ff932facd9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.108Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"d2b2cffb-2c27-4160-9b06-eed7de59e1b1","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:19:53.117Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"d2b2cffb-2c27-4160-9b06-eed7de59e1b1","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:19:53.117Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"d2b2cffb-2c27-4160-9b06-eed7de59e1b1","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3"}
{"level":"info","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3"}
{"level":"info","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3"}
{"level":"error","ts":"2026-06-10T10:19:53.199Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef5135b-56f2-446c-9100-1acdd2056aa3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59"}
{"level":"info","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59"}
{"level":"info","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59"}
{"level":"error","ts":"2026-06-10T10:19:53.280Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"c4c452d7-2ca6-4db2-9950-21223cd3ea59","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980"}
{"level":"info","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980"}
{"level":"info","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980","error":"head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"error","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found"}
{"level":"info","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980"}
{"level":"error","ts":"2026-06-10T10:19:53.499Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"63b57d39-b05b-47e6-8e7e-7b3b145b5980","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster-auth not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.818Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","name":"kube-rbac-proxy-config-raycluster-auth","operation":"created"}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed"}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed"}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed"}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"Waiting for authentication resources to be ready","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","condition":"nil","reason":"AuthenticationPending","conditionStale":false,"observedGeneration":0,"currentGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"error","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:19:53.820Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"}]}
{"level":"info","ts":"2026-06-10T10:19:53.821Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-06-10T10:19:53.823Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.823Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7"}
{"level":"error","ts":"2026-06-10T10:19:53.823Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c78c6cae-ffe9-4c81-a63c-09d51056cfa7","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.823Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"167b620c-faa1-44e3-9bad-16b785558923","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"}}
{"level":"info","ts":"2026-06-10T10:19:53.823Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"167b620c-faa1-44e3-9bad-16b785558923","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.823Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"167b620c-faa1-44e3-9bad-16b785558923","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.833Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"167b620c-faa1-44e3-9bad-16b785558923","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.833Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"00d472c2-f57a-4947-9e2e-ad217c3efd34","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"}}
{"level":"info","ts":"2026-06-10T10:19:53.833Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"00d472c2-f57a-4947-9e2e-ad217c3efd34","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.833Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"00d472c2-f57a-4947-9e2e-ad217c3efd34","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.834Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"3cb3f06a-a44b-4944-b8b3-592612f5af79","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.899Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"299d7524-f368-4a7d-98ae-3834fd4dbac9","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.899Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"299d7524-f368-4a7d-98ae-3834fd4dbac9"}
{"level":"info","ts":"2026-06-10T10:19:53.899Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"00d472c2-f57a-4947-9e2e-ad217c3efd34","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:53.906Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-qqsjt","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-10T10:19:53.906Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed"}
{"level":"error","ts":"2026-06-10T10:19:53.906Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2230707c-3d6f-4d96-8ac4-c108533609ed","error":"FailedCreateHeadPod\nwaiting for AuthenticationReady condition: Waiting for AuthenticationController to create authentication resources","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:53.910Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"3cb3f06a-a44b-4944-b8b3-592612f5af79","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:19:53.920Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"3cb3f06a-a44b-4944-b8b3-592612f5af79","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:19:53.920Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"3cb3f06a-a44b-4944-b8b3-592612f5af79","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b"}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b"}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b"}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b"}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:19:54.547Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","enabled":true}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","name":"raycluster-auth-head-jc44v"}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b"}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:19:54.565Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","enabled":true}
{"level":"info","ts":"2026-06-10T10:19:54.577Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","name":"raycluster-auth-small-group-worker-v9xvs"}
{"level":"info","ts":"2026-06-10T10:19:54.577Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.577Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.577Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-auth-head-svc"},"newHead":{"podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-10T10:19:54.577Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"72305714-b099-40b2-bc58-5c25164e635b","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:19:54.590Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3"}
{"level":"info","ts":"2026-06-10T10:19:54.590Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3"}
{"level":"info","ts":"2026-06-10T10:19:54.590Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"de850464-f447-45c0-af6b-1f996543cab6","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:54.590Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:54.590Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"de850464-f447-45c0-af6b-1f996543cab6","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:19:54.590Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"9e50c418-4e50-4edd-99ec-769666e4ef2f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:54.590Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:19:54.591Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"b939c692-6a6b-4fb9-a57a-36f8846083b3","seconds":300}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0"}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0"}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:19:54.596Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"e6e27ce3-cf4b-4ae7-b175-962f867ce1f0","seconds":300}
{"level":"info","ts":"2026-06-10T10:19:54.602Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"9e50c418-4e50-4edd-99ec-769666e4ef2f","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5"}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5"}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:19:54.610Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8c0c9996-5cf6-4a6e-8abc-ad116c6a30d5","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:19:54.616Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"9e50c418-4e50-4edd-99ec-769666e4ef2f","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:19:54.616Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"9e50c418-4e50-4edd-99ec-769666e4ef2f","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"ca463a41-fb60-4cd8-9d41-809d60eb5822","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"87c2c919-a7d2-4c96-951b-bd49d6d20773","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"ca463a41-fb60-4cd8-9d41-809d60eb5822","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:19:54.621Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:19:54.628Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","name":"raycluster-auth","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-qqsjt","name":"raycluster-auth"}}
{"level":"info","ts":"2026-06-10T10:19:54.628Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80"}
{"level":"error","ts":"2026-06-10T10:19:54.628Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4cd0ad14-97de-4aa2-ae6b-8622655c2c80","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:19:54.632Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"87c2c919-a7d2-4c96-951b-bd49d6d20773","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2"}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2"}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:19:54.634Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5dbe73d5-e571-464d-9586-451386a13fe2","seconds":300}
{"level":"info","ts":"2026-06-10T10:19:54.640Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"87c2c919-a7d2-4c96-951b-bd49d6d20773","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:19:54.640Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"87c2c919-a7d2-4c96-951b-bd49d6d20773","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e"}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e"}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:19:55.013Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"71854d29-46e1-4ade-a845-f411115fee5e","seconds":300}
{"level":"info","ts":"2026-06-10T10:19:55.025Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed"}
{"level":"info","ts":"2026-06-10T10:19:55.025Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed"}
{"level":"info","ts":"2026-06-10T10:19:55.025Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:19:55.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:19:55.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:55.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:19:55.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:19:55.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:55.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:19:55.026Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:19:55.026Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:55.026Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:19:55.026Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:19:55.026Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"5ad44ab9-cb21-4b99-bdb6-d43c015970ed","seconds":300}
{"level":"info","ts":"2026-06-10T10:20:22.399Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8f456de5-383c-41af-b5d4-02b2388d2d83","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:20:22.400Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8f456de5-383c-41af-b5d4-02b2388d2d83","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:21:22.400Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"eb8c51a7-8b69-447b-a5a5-e56d257ed9b7","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:21:22.401Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"eb8c51a7-8b69-447b-a5a5-e56d257ed9b7","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:22:22.401Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"71b32535-60d4-4b81-8071-57180f33de60","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:22:22.401Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"71b32535-60d4-4b81-8071-57180f33de60","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:23:22.402Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"67e6d9fb-6dee-4024-8064-2feb986d2bd0","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:23:22.402Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"67e6d9fb-6dee-4024-8064-2feb986d2bd0","rayCluster":"raycluster-auth","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890"}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890"}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"newHead":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-10T10:23:54.736Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"94c26cce-e92c-4206-bd3c-65082076b890","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:19:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:23:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:23:54.758Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8770612a-6f0a-41ec-9708-33e147aa0a77","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:23:54.758Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8a86c3a4-8a1a-49c3-ae46-0968614ac581","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:23:54.758Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8770612a-6f0a-41ec-9708-33e147aa0a77","pod":"raycluster-auth-head-jc44v","ip":"10.134.0.15"}
{"level":"info","ts":"2026-06-10T10:23:54.788Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8a86c3a4-8a1a-49c3-ae46-0968614ac581","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:23:54.803Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8a86c3a4-8a1a-49c3-ae46-0968614ac581","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:23:54.804Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8a86c3a4-8a1a-49c3-ae46-0968614ac581","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:23:54.827Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8770612a-6f0a-41ec-9708-33e147aa0a77","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-qqsjt.svc","raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local"],"ipAddresses":["10.134.0.15","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:23:54.845Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8770612a-6f0a-41ec-9708-33e147aa0a77","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-qqsjt.svc","raycluster-auth-worker-svc.test-ns-qqsjt.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-qqsjt.svc","*.raycluster-auth-worker-svc.test-ns-qqsjt.svc.cluster.local","*-worker-*.test-ns-qqsjt.svc","*-worker-*.test-ns-qqsjt.svc.cluster.local"],"ipAddresses":["10.134.0.15","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:23:54.845Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8770612a-6f0a-41ec-9708-33e147aa0a77","rayCluster":"raycluster-auth","podIPs":["10.134.0.15"]}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e"}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e"}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:23:55.096Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:23:55.097Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:23:55.097Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1734fce4-e5f4-487f-8478-23ffd7b1900e","seconds":300}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4"}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4"}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:23:56.758Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bef138f7-9819-47d6-ad35-4d89f182dab4","seconds":300}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225"}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225"}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:23:59.717Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ab1313a6-1658-47b2-939e-a84f3d62e225","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7"}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7"}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:04.096Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"69db2e8b-a7a2-4cad-bb32-09e058b360b7","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c"}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c"}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:05.110Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:05.111Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:05.111Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:24:05.111Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"ffa624cd-f865-462f-9780-a2468f22a80c","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:23:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:05.123Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"2039d766-a4fe-4487-8e30-cf4c8aa90ca9","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:05.123Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"4f384ba1-f816-4860-9b7a-6f58547ed5ed","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:05.124Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"4f384ba1-f816-4860-9b7a-6f58547ed5ed","pod":"raycluster-auth-head-jc44v","ip":"10.134.0.15"}
{"level":"info","ts":"2026-06-10T10:24:05.124Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"4f384ba1-f816-4860-9b7a-6f58547ed5ed","pod":"raycluster-auth-small-group-worker-v9xvs","ip":"10.133.0.12"}
{"level":"info","ts":"2026-06-10T10:24:05.134Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"4f384ba1-f816-4860-9b7a-6f58547ed5ed","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-qqsjt.svc","raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local"],"ipAddresses":["10.134.0.15","10.133.0.12","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:24:05.136Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"2039d766-a4fe-4487-8e30-cf4c8aa90ca9","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:24:05.147Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"4f384ba1-f816-4860-9b7a-6f58547ed5ed","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-qqsjt.svc","raycluster-auth-worker-svc.test-ns-qqsjt.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-qqsjt.svc","*.raycluster-auth-worker-svc.test-ns-qqsjt.svc.cluster.local","*-worker-*.test-ns-qqsjt.svc","*-worker-*.test-ns-qqsjt.svc.cluster.local"],"ipAddresses":["10.134.0.15","10.133.0.12","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:24:05.147Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"4f384ba1-f816-4860-9b7a-6f58547ed5ed","rayCluster":"raycluster-auth","podIPs":["10.134.0.15","10.133.0.12"]}
{"level":"info","ts":"2026-06-10T10:24:05.150Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"2039d766-a4fe-4487-8e30-cf4c8aa90ca9","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:24:05.150Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"2039d766-a4fe-4487-8e30-cf4c8aa90ca9","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389"}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389"}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:07.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:07.125Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:07.125Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:07.125Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:07.125Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"6c010bd4-e00f-4f71-9320-2fda331dc389","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc"}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc"}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","podName":"raycluster-auth-head-jc44v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:19:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","podName":"raycluster-auth-head-jc44v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:19:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:24:07.764Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"4ab75016-d56c-4d46-9721-059ea9bd29bc","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:07.776Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"823e51e0-8a06-4279-8079-9b7b11e594ef","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:07.776Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6447fbf9-6902-4973-a62e-4cfb5faaff9b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:07.776Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"823e51e0-8a06-4279-8079-9b7b11e594ef","pod":"raycluster-auth-head-jc44v","ip":"10.134.0.15"}
{"level":"info","ts":"2026-06-10T10:24:07.776Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"823e51e0-8a06-4279-8079-9b7b11e594ef","pod":"raycluster-auth-small-group-worker-v9xvs","ip":"10.133.0.12"}
{"level":"info","ts":"2026-06-10T10:24:07.777Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"823e51e0-8a06-4279-8079-9b7b11e594ef","rayCluster":"raycluster-auth","podIPs":["10.134.0.15","10.133.0.12"]}
{"level":"info","ts":"2026-06-10T10:24:07.788Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6447fbf9-6902-4973-a62e-4cfb5faaff9b","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:24:07.800Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6447fbf9-6902-4973-a62e-4cfb5faaff9b","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:24:07.800Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6447fbf9-6902-4973-a62e-4cfb5faaff9b","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff"}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff"}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","podName":"raycluster-auth-head-jc44v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:19:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","podName":"raycluster-auth-head-jc44v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:19:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:09.778Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"30c2103f-2987-4301-974b-b3c0803e71ff","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:13.790Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde"}
{"level":"info","ts":"2026-06-10T10:24:13.790Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde"}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","podName":"raycluster-auth-small-group-worker-v9xvs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:19:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","podName":"raycluster-auth-small-group-worker-v9xvs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:19:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:24:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:24:13.791Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d6d7ef39-d5a6-4e73-9a6b-ed9e08e6acde","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:24:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:13.802Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"56b9b77d-ce3a-4cde-abed-59d1ec458ff8","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:13.802Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"403c8534-3abb-452c-af65-1b53fa550199","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:13.802Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"56b9b77d-ce3a-4cde-abed-59d1ec458ff8","pod":"raycluster-auth-head-jc44v","ip":"10.134.0.15"}
{"level":"info","ts":"2026-06-10T10:24:13.803Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"56b9b77d-ce3a-4cde-abed-59d1ec458ff8","pod":"raycluster-auth-small-group-worker-v9xvs","ip":"10.133.0.12"}
{"level":"info","ts":"2026-06-10T10:24:13.803Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"56b9b77d-ce3a-4cde-abed-59d1ec458ff8","rayCluster":"raycluster-auth","podIPs":["10.134.0.15","10.133.0.12"]}
{"level":"info","ts":"2026-06-10T10:24:13.813Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"403c8534-3abb-452c-af65-1b53fa550199","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:24:13.824Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"403c8534-3abb-452c-af65-1b53fa550199","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:24:13.824Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"403c8534-3abb-452c-af65-1b53fa550199","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d"}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d"}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","podName":"raycluster-auth-small-group-worker-v9xvs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:19:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","podName":"raycluster-auth-small-group-worker-v9xvs","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:19:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:15.803Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"8e12753c-8f1b-42e3-8140-e1333dae218d","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c"}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c"}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:24:16.137Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"329c50aa-6cad-4904-b47a-2149d5705f3c","name":"raycluster-auth","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:24:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:16Z","stateTransitionTimes":{"ready":"2026-06-10T10:24:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:24:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:16.152Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"841bb100-2183-449c-b7a0-56d9f12a51b1","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:16.152Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"9e2fc891-7920-4d2a-b876-7773806f09cb","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:16.153Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"9e2fc891-7920-4d2a-b876-7773806f09cb","pod":"raycluster-auth-head-jc44v","ip":"10.134.0.15"}
{"level":"info","ts":"2026-06-10T10:24:16.153Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"9e2fc891-7920-4d2a-b876-7773806f09cb","pod":"raycluster-auth-small-group-worker-v9xvs","ip":"10.133.0.12"}
{"level":"info","ts":"2026-06-10T10:24:16.153Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"9e2fc891-7920-4d2a-b876-7773806f09cb","rayCluster":"raycluster-auth","podIPs":["10.134.0.15","10.133.0.12"]}
{"level":"info","ts":"2026-06-10T10:24:16.166Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"841bb100-2183-449c-b7a0-56d9f12a51b1","name":"raycluster-auth-head"}
{"level":"info","ts":"2026-06-10T10:24:16.183Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"841bb100-2183-449c-b7a0-56d9f12a51b1","name":"raycluster-auth-workers"}
{"level":"info","ts":"2026-06-10T10:24:16.183Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"841bb100-2183-449c-b7a0-56d9f12a51b1","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:16.687Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.687Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-ccmjv"}
{"level":"info","ts":"2026-06-10T10:24:16.703Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","namespacedName":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:24:16.703Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849"}
{"level":"info","ts":"2026-06-10T10:24:16.703Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849"}
{"level":"info","ts":"2026-06-10T10:24:16.703Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.703Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.703Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.703Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.703Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.703Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"3021d853-fd87-4351-a75f-626c9dbb6916","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.711Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.711Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster","namespace":"test-ns-ccmjv"}
{"level":"info","ts":"2026-06-10T10:24:16.714Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","name":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:16.714Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-10T10:24:16.714Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.714Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-10T10:24:16.714Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.714Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.714Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.714Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:24:16.714Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:16.726Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.726Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.728Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"3021d853-fd87-4351-a75f-626c9dbb6916","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.729Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"375259e1-4330-4fae-9027-c57f0ce01164","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.736Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","name":"raycluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccmjv","name":"raycluster"}}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849"}
{"level":"error","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d6e5cca4-ef27-4865-ae71-7d6e9a4fc849","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e"}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e"}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:24:16.737Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:16.799Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6"}
{"level":"info","ts":"2026-06-10T10:24:16.799Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","name":"kuberay-gateway-access","namespace":"test-ns-ccmjv","operation":"created"}
{"level":"info","ts":"2026-06-10T10:24:16.799Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6","rayCluster":"raycluster","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:24:16.799Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.799Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e"}
{"level":"error","ts":"2026-06-10T10:24:16.799Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"60424ec9-d508-41a8-9c17-891ee94d313e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160"}
{"level":"info","ts":"2026-06-10T10:24:16.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160"}
{"level":"info","ts":"2026-06-10T10:24:16.800Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:16.800Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-10T10:24:16.800Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.800Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-10T10:24:16.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.800Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:24:16.800Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:16.805Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"375259e1-4330-4fae-9027-c57f0ce01164","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccmjv","name":"raycluster"}}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160"}
{"level":"error","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"46f56cd1-7179-4e27-947f-1e93b8943160","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d"}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d"}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d"}
{"level":"error","ts":"2026-06-10T10:24:16.813Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"3df11cca-66ac-411b-a75e-3dc93ee14c0d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.814Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.814Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","name":"test-ns-ccmjv-raycluster","operation":"created"}
{"level":"info","ts":"2026-06-10T10:24:16.825Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","name":"kube-rbac-proxy-config-raycluster","operation":"created"}
{"level":"info","ts":"2026-06-10T10:24:16.826Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"375259e1-4330-4fae-9027-c57f0ce01164","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:24:16.826Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"375259e1-4330-4fae-9027-c57f0ce01164","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.826Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"067cb1c3-c293-4ef2-835c-8f058d5f1419","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.833Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de"}
{"level":"info","ts":"2026-06-10T10:24:16.833Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de"}
{"level":"info","ts":"2026-06-10T10:24:16.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:16.834Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-10T10:24:16.834Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.834Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-10T10:24:16.834Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6"}
{"level":"info","ts":"2026-06-10T10:24:16.834Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:24:16.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.899Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de"}
{"level":"error","ts":"2026-06-10T10:24:16.899Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"4169fa1d-9b0b-4e11-b91a-6d2932fcf6de","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:24:16.899Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.899Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b"}
{"level":"error","ts":"2026-06-10T10:24:16.899Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"8fe76141-0e9d-4551-a58c-9b884ab9532b","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.900Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a7c5705a-036c-46c8-a704-869b8ed505c3","namespacedName":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:24:16.900Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6"}
{"level":"info","ts":"2026-06-10T10:24:16.900Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a7c5705a-036c-46c8-a704-869b8ed505c3","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.900Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a7c5705a-036c-46c8-a704-869b8ed505c3","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.900Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:24:16.900Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6c3d0c98-d868-4233-adc7-8f7317a765a6"}
{"level":"info","ts":"2026-06-10T10:24:16.900Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f0a5b894-2549-4ffe-b048-66ebdb27a3fe","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:16.900Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f0a5b894-2549-4ffe-b048-66ebdb27a3fe"}
{"level":"info","ts":"2026-06-10T10:24:16.907Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"067cb1c3-c293-4ef2-835c-8f058d5f1419","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52"}
{"level":"info","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52"}
{"level":"info","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52"}
{"level":"error","ts":"2026-06-10T10:24:16.999Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"db7b1427-f585-4ade-b1d3-f982d26a0a52","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:17.000Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d4904fee-6812-427d-87fa-491af2d9ea44","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:17.000Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d4904fee-6812-427d-87fa-491af2d9ea44"}
{"level":"info","ts":"2026-06-10T10:24:17.000Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a7c5705a-036c-46c8-a704-869b8ed505c3","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:17.000Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d1737a8f-aba6-429b-b57e-d8df4fb4dd86","namespacedName":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:24:17.000Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d1737a8f-aba6-429b-b57e-d8df4fb4dd86","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:17.000Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d1737a8f-aba6-429b-b57e-d8df4fb4dd86","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:17.018Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"067cb1c3-c293-4ef2-835c-8f058d5f1419","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:24:17.018Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"067cb1c3-c293-4ef2-835c-8f058d5f1419","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:17.018Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"36834acf-8938-486e-9382-1aec1b717b55","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:17.021Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d1737a8f-aba6-429b-b57e-d8df4fb4dd86","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:17.035Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"36834acf-8938-486e-9382-1aec1b717b55","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:24:17.092Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"36834acf-8938-486e-9382-1aec1b717b55","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:24:17.092Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"36834acf-8938-486e-9382-1aec1b717b55","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db"}
{"level":"info","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db"}
{"level":"info","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db"}
{"level":"error","ts":"2026-06-10T10:24:17.160Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9121f0aa-a7a9-4067-ab84-7a38e55c46db","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d"}
{"level":"info","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d"}
{"level":"info","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d"}
{"level":"error","ts":"2026-06-10T10:24:17.482Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5748d717-8af3-46ea-b715-114957d95e9d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d"}
{"level":"info","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d"}
{"level":"info","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d","error":"head certificate secret ray-head-secret-raycluster not found"}
{"level":"error","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found"}
{"level":"info","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d","filter labels":["test-ns-ccmjv",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d"}
{"level":"error","ts":"2026-06-10T10:24:18.123Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5ad6e5f5-43b7-45ad-8c62-96c966b7c60d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-raycluster not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b"}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b"}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:18.153Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d3291e05-6562-4153-a025-e1aa5afd359b","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9"}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9"}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9"}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9"}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-ccmjv.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-head-svc.test-ns-ccmjv.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:24:19.404Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","enabled":true}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","name":"raycluster-head-jjvtb"}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9"}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:19.429Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","enabled":true}
{"level":"info","ts":"2026-06-10T10:24:19.443Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","name":"raycluster-small-group-worker-7j6bp"}
{"level":"info","ts":"2026-06-10T10:24:19.443Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.443Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-head-svc"},"newHead":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-06-10T10:24:19.443Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b97ef0ce-b24d-4fce-9b74-900a717fcca9","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:19.457Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243"}
{"level":"info","ts":"2026-06-10T10:24:19.457Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243"}
{"level":"info","ts":"2026-06-10T10:24:19.457Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f99475d8-ee53-4c6f-80f5-381592e16f1b","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"raycluster-head-svc"},"newHead":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"09006de2-178d-4996-b903-9097ec285022","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:19.458Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f99475d8-ee53-4c6f-80f5-381592e16f1b","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccmjv","name":"raycluster"}}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243"}
{"level":"error","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"188d2d7e-e61b-4a69-9518-a1d4acde6243","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3"}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3"}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:19.465Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"e5bd987d-b587-47e8-b436-d8fecb9833a3","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:19.470Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca"}
{"level":"info","ts":"2026-06-10T10:24:19.470Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca"}
{"level":"info","ts":"2026-06-10T10:24:19.470Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:19.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:19.471Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"d532ceb5-9078-4242-99a1-60f5f8b805ca","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:19.473Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"09006de2-178d-4996-b903-9097ec285022","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533"}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533"}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:24:19.499Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"38c2e48d-b4b8-46cd-b8fe-78b45aaed533","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:19.503Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"09006de2-178d-4996-b903-9097ec285022","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:24:19.503Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"09006de2-178d-4996-b903-9097ec285022","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"1e45654b-7514-4fde-b71f-a5c56df7b8c3","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d9066028-d922-4658-83e8-a23c6cc79dd9","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d9066028-d922-4658-83e8-a23c6cc79dd9","rayCluster":"raycluster","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:24:19.511Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:19.516Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-ccmjv","name":"raycluster"}}
{"level":"info","ts":"2026-06-10T10:24:19.516Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d"}
{"level":"error","ts":"2026-06-10T10:24:19.517Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"993faa9d-3a9d-4029-8eeb-ff75cf1f2d7d","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:19.521Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"1e45654b-7514-4fde-b71f-a5c56df7b8c3","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b"}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b"}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:19.522Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f5774a3b-f149-4e10-9421-43f5c341828b","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:19.532Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"1e45654b-7514-4fde-b71f-a5c56df7b8c3","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:24:19.532Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"1e45654b-7514-4fde-b71f-a5c56df7b8c3","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:19.877Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be"}
{"level":"info","ts":"2026-06-10T10:24:19.877Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be"}
{"level":"info","ts":"2026-06-10T10:24:19.877Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:19.877Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:19.878Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"507e3eb0-c1dd-4972-ba37-91fed062c1be","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6"}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6"}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:19.898Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"49156b73-93c0-4e85-8f94-568dbfa59ec6","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4"}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4"}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"newHead":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"}}
{"level":"info","ts":"2026-06-10T10:24:20.145Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"5e9efd73-357f-45db-91d9-1cc974f2e0b4","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:20.155Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"56120646-7925-4cda-80f4-8802d57a8c69","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:20.155Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"9f45174a-6ffe-4857-88ce-7f4930a64396","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:20.155Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"56120646-7925-4cda-80f4-8802d57a8c69","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:24:20.166Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"9f45174a-6ffe-4857-88ce-7f4930a64396","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:24:20.167Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"56120646-7925-4cda-80f4-8802d57a8c69","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-ccmjv.svc","raycluster-head-svc.test-ns-ccmjv.svc.cluster.local"],"ipAddresses":["10.133.0.13","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:24:20.178Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"9f45174a-6ffe-4857-88ce-7f4930a64396","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:24:20.178Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"9f45174a-6ffe-4857-88ce-7f4930a64396","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:20.178Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"56120646-7925-4cda-80f4-8802d57a8c69","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-ccmjv.svc","raycluster-worker-svc.test-ns-ccmjv.svc.cluster.local","*.raycluster-worker-svc.test-ns-ccmjv.svc","*.raycluster-worker-svc.test-ns-ccmjv.svc.cluster.local","*-worker-*.test-ns-ccmjv.svc","*-worker-*.test-ns-ccmjv.svc.cluster.local"],"ipAddresses":["10.133.0.13","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:24:20.178Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"56120646-7925-4cda-80f4-8802d57a8c69","rayCluster":"raycluster","podIPs":["10.133.0.13"]}
{"level":"info","ts":"2026-06-10T10:24:21.633Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f"}
{"level":"info","ts":"2026-06-10T10:24:21.633Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f"}
{"level":"info","ts":"2026-06-10T10:24:21.633Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:21.633Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:21.633Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:21.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:21.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:21.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:21.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:21.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:21.634Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:21.634Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"340cf425-8007-4c83-b6a6-5ed79af6702f","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:21.638Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865"}
{"level":"info","ts":"2026-06-10T10:24:21.638Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865"}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:21.639Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2b972dda-a57a-49a0-a38e-2669ff404865","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8"}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8"}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:21.659Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:21.660Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:21.660Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"42698e7a-b486-426d-be50-36c86b5bdca8","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad"}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad"}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-small-group-worker-v9xvs. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:21.662Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3ef331bb-4ce8-440d-a673-bf3d735ea8ad","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:22.156Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549"}
{"level":"info","ts":"2026-06-10T10:24:22.156Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549"}
{"level":"info","ts":"2026-06-10T10:24:22.156Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:22.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:22.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:22.157Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:22.157Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:22.157Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:22.157Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:24:22.157Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:22.157Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:22.157Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:24:22.157Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:22.157Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"b32a62da-f005-429d-9b37-bc5ff2b25549","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:22.202Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25"}
{"level":"info","ts":"2026-06-10T10:24:22.202Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25"}
{"level":"info","ts":"2026-06-10T10:24:22.202Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:22.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:22.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:22.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:22.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","worker Pod":"raycluster-auth-small-group-worker-v9xvs","shouldDelete":true,"reason":"The worker Pod raycluster-auth-small-group-worker-v9xvs status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-10T10:24:22.206Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:22.206Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-06-10T10:24:22.206Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:24:22.206Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:16Z","stateTransitionTimes":{"ready":"2026-06-10T10:24:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:24:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:22Z","stateTransitionTimes":{"ready":"2026-06-10T10:24:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:24:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:22.218Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25"}
{"level":"error","ts":"2026-06-10T10:24:22.218Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f691f828-0cdc-4107-9647-1d1292420c25","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:22.218Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c6900b6c-ad69-405d-9d85-ad9873b2afd4","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:22.218Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8f0b71aa-c539-43cf-a470-ae86c7b09dfa","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:22.299Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c6900b6c-ad69-405d-9d85-ad9873b2afd4","pod":"raycluster-auth-head-jc44v","ip":"10.134.0.15"}
{"level":"info","ts":"2026-06-10T10:24:22.299Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c6900b6c-ad69-405d-9d85-ad9873b2afd4","pod":"raycluster-auth-small-group-worker-v9xvs","ip":"10.133.0.12"}
{"level":"info","ts":"2026-06-10T10:24:22.299Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c6900b6c-ad69-405d-9d85-ad9873b2afd4","rayCluster":"raycluster-auth","podIPs":["10.134.0.15","10.133.0.12"]}
{"level":"info","ts":"2026-06-10T10:24:22.299Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d15a40dd-aee2-4d82-b28d-817b6c2e5ff8"}
{"level":"info","ts":"2026-06-10T10:24:22.299Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d15a40dd-aee2-4d82-b28d-817b6c2e5ff8"}
{"level":"info","ts":"2026-06-10T10:24:22.299Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d15a40dd-aee2-4d82-b28d-817b6c2e5ff8","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:22.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d15a40dd-aee2-4d82-b28d-817b6c2e5ff8","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:24:22.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d15a40dd-aee2-4d82-b28d-817b6c2e5ff8","head Pod":"raycluster-auth-head-jc44v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-head-jc44v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:24:22.305Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"8f0b71aa-c539-43cf-a470-ae86c7b09dfa","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:22.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d15a40dd-aee2-4d82-b28d-817b6c2e5ff8","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-10T10:24:22.399Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d15a40dd-aee2-4d82-b28d-817b6c2e5ff8","podName":"raycluster-auth-small-group-worker-v9xvs","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-10T10:24:22.399Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d15a40dd-aee2-4d82-b28d-817b6c2e5ff8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:22.399Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"d15a40dd-aee2-4d82-b28d-817b6c2e5ff8","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:22.403Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"94d2a0e6-c61e-4051-ac57-6723860bf05f","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:22.403Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"94d2a0e6-c61e-4051-ac57-6723860bf05f","pod":"raycluster-auth-head-jc44v","ip":"10.134.0.15"}
{"level":"info","ts":"2026-06-10T10:24:22.403Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"94d2a0e6-c61e-4051-ac57-6723860bf05f","pod":"raycluster-auth-small-group-worker-v9xvs","ip":"10.133.0.12"}
{"level":"info","ts":"2026-06-10T10:24:22.403Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"94d2a0e6-c61e-4051-ac57-6723860bf05f","rayCluster":"raycluster-auth","podIPs":["10.134.0.15","10.133.0.12"]}
{"level":"info","ts":"2026-06-10T10:24:22.894Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122"}
{"level":"info","ts":"2026-06-10T10:24:22.894Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122"}
{"level":"info","ts":"2026-06-10T10:24:22.894Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:22.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122","Found 1 head Pod":"raycluster-auth-head-jc44v","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-10 10:23:59 +0000 UTC,FinishedAt:2026-06-10 10:24:22 +0000 UTC,ContainerID:cri-o://ebcffea30b0c6a1fb33a48b57b63d66a0076a2a263b81e288abc1dc213fb59df,}"}
{"level":"info","ts":"2026-06-10T10:24:22.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122","head Pod":"raycluster-auth-head-jc44v","shouldDelete":true,"reason":"The head Pod raycluster-auth-head-jc44v status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-10T10:24:22.898Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod raycluster-auth-head-jc44v status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:22.898Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122","error":"The head Pod raycluster-auth-head-jc44v status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-10T10:24:22.898Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:24:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:22Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:24:22.898Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:22Z","stateTransitionTimes":{"ready":"2026-06-10T10:24:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:24:13Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:22Z","stateTransitionTimes":{"ready":"2026-06-10T10:24:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:22Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:22.908Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122"}
{"level":"error","ts":"2026-06-10T10:24:22.908Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"1e8b74dd-996b-4c47-b008-5163f56d7122","error":"The head Pod raycluster-auth-head-jc44v status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:22.909Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"017e39d6-02ea-4adf-ac96-bb7b48b3fe4f","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:22.909Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"349d48f0-ff0d-43e9-8f3d-df0e3c61d501","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:22.909Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"017e39d6-02ea-4adf-ac96-bb7b48b3fe4f","pod":"raycluster-auth-small-group-worker-v9xvs","ip":"10.133.0.12"}
{"level":"info","ts":"2026-06-10T10:24:22.909Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"017e39d6-02ea-4adf-ac96-bb7b48b3fe4f","pod":"raycluster-auth-head-jc44v","ip":"10.134.0.15"}
{"level":"info","ts":"2026-06-10T10:24:22.909Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"017e39d6-02ea-4adf-ac96-bb7b48b3fe4f","rayCluster":"raycluster-auth","podIPs":["10.133.0.12","10.134.0.15"]}
{"level":"info","ts":"2026-06-10T10:24:22.913Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"349d48f0-ff0d-43e9-8f3d-df0e3c61d501","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:22.914Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9db816cf-fcd6-4db2-aaba-c1982e73d7cc"}
{"level":"info","ts":"2026-06-10T10:24:22.914Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9db816cf-fcd6-4db2-aaba-c1982e73d7cc"}
{"level":"info","ts":"2026-06-10T10:24:22.914Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9db816cf-fcd6-4db2-aaba-c1982e73d7cc","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:22.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9db816cf-fcd6-4db2-aaba-c1982e73d7cc","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:22.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9db816cf-fcd6-4db2-aaba-c1982e73d7cc","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-10T10:24:22.914Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9db816cf-fcd6-4db2-aaba-c1982e73d7cc","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-10T10:24:22.915Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9db816cf-fcd6-4db2-aaba-c1982e73d7cc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:22.915Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9db816cf-fcd6-4db2-aaba-c1982e73d7cc","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:23.168Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"36cc95f2-dbc3-478d-9f81-72ddb60dbb68"}
{"level":"info","ts":"2026-06-10T10:24:23.168Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"36cc95f2-dbc3-478d-9f81-72ddb60dbb68"}
{"level":"info","ts":"2026-06-10T10:24:23.168Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"36cc95f2-dbc3-478d-9f81-72ddb60dbb68","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.169Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"36cc95f2-dbc3-478d-9f81-72ddb60dbb68","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:23.169Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"36cc95f2-dbc3-478d-9f81-72ddb60dbb68","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-10T10:24:23.169Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"36cc95f2-dbc3-478d-9f81-72ddb60dbb68","podName":"raycluster-auth-head-jc44v","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-10T10:24:23.169Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"36cc95f2-dbc3-478d-9f81-72ddb60dbb68","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:24:23.169Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"36cc95f2-dbc3-478d-9f81-72ddb60dbb68","seconds":300}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5"}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5"}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5"}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.173Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:23.177Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.177Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.177Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:22Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:22Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:24:23Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}]}
{"level":"info","ts":"2026-06-10T10:24:23.177Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:22Z","stateTransitionTimes":{"ready":"2026-06-10T10:24:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:22Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:23Z","stateTransitionTimes":{"ready":"2026-06-10T10:24:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:22Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:24:23Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:23.189Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5"}
{"level":"error","ts":"2026-06-10T10:24:23.189Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"0aa048b7-81f4-461f-bfbc-c1d1ce49aac5","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.189Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"78339c6d-a6c1-40a8-8b76-60264e18c081","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:23.189Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"e780f9e6-4cab-4d9f-9cee-4a46ad616292","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:23.189Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"78339c6d-a6c1-40a8-8b76-60264e18c081","pod":"raycluster-auth-head-jc44v","ip":"10.134.0.15"}
{"level":"info","ts":"2026-06-10T10:24:23.194Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"e780f9e6-4cab-4d9f-9cee-4a46ad616292","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e"}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e"}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e"}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.199Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","enabled":true}
{"level":"info","ts":"2026-06-10T10:24:23.200Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"78339c6d-a6c1-40a8-8b76-60264e18c081","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-qqsjt.svc","raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local"],"ipAddresses":["10.134.0.15","127.0.0.1"]}
{"level":"error","ts":"2026-06-10T10:24:23.202Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.202Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.202Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e"}
{"level":"error","ts":"2026-06-10T10:24:23.202Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"808128be-6eed-4e35-8a5e-b2e80bdc958e","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.211Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"78339c6d-a6c1-40a8-8b76-60264e18c081","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-qqsjt.svc","raycluster-auth-worker-svc.test-ns-qqsjt.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-qqsjt.svc","*.raycluster-auth-worker-svc.test-ns-qqsjt.svc.cluster.local","*-worker-*.test-ns-qqsjt.svc","*-worker-*.test-ns-qqsjt.svc.cluster.local"],"ipAddresses":["10.134.0.15","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:24:23.211Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"78339c6d-a6c1-40a8-8b76-60264e18c081","rayCluster":"raycluster-auth","podIPs":["10.134.0.15"]}
{"level":"info","ts":"2026-06-10T10:24:23.213Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d"}
{"level":"info","ts":"2026-06-10T10:24:23.213Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d"}
{"level":"info","ts":"2026-06-10T10:24:23.213Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d"}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.214Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:23.217Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.217Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.217Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d"}
{"level":"error","ts":"2026-06-10T10:24:23.217Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9df1eabc-e7da-41a3-841c-b82a03ea755d","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.237Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41"}
{"level":"info","ts":"2026-06-10T10:24:23.237Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41"}
{"level":"info","ts":"2026-06-10T10:24:23.237Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41"}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.238Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:23.241Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.241Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.242Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41"}
{"level":"error","ts":"2026-06-10T10:24:23.242Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"9392174d-068d-453c-a29b-35def4332a41","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.282Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3"}
{"level":"info","ts":"2026-06-10T10:24:23.283Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3"}
{"level":"info","ts":"2026-06-10T10:24:23.283Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3"}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.299Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:23.304Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.304Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.304Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3"}
{"level":"error","ts":"2026-06-10T10:24:23.304Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2dafd779-1d9b-4dfb-b394-b835c97fd7e3","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5"}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5"}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5"}
{"level":"info","ts":"2026-06-10T10:24:23.385Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.386Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.386Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.386Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:23.389Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.390Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.390Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5"}
{"level":"error","ts":"2026-06-10T10:24:23.390Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"408c2ad3-7fef-4eda-a689-8b64cfeeaad5","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9"}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9"}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9"}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.550Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:23.555Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.555Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.555Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9"}
{"level":"error","ts":"2026-06-10T10:24:23.555Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"f26262fc-9fbd-481d-bc8a-605b1f5810d9","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146"}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146"}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146"}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","rayStartCmd":"ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:24:23.818Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:23.821Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.821Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.822Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146"}
{"level":"error","ts":"2026-06-10T10:24:23.822Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"2c50024a-1cf2-433f-82d5-e87f15801146","error":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff"}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff"}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff"}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff"}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:24:23.823Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:23.826Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.826Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.826Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:23.826Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:23.826Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:23.826Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"newHead":{"serviceName":"raycluster-auth-head-svc"}}
{"level":"info","ts":"2026-06-10T10:24:23.826Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","name":"raycluster-auth","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:23Z","stateTransitionTimes":{"ready":"2026-06-10T10:24:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.15","serviceIP":"10.134.0.15","podName":"raycluster-auth-head-jc44v","serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:22Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:24:23Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"raycluster-auth-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:23Z","stateTransitionTimes":{"ready":"2026-06-10T10:24:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:22Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:19:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:19:53Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:24:23Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:24:23.837Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff"}
{"level":"error","ts":"2026-06-10T10:24:23.837Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"805d49d1-012c-4570-976b-f5542386b0ff","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.838Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6ca676bd-9770-4f85-9c65-b4590da6aeb9","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:23.838Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"67052bbc-fecc-40cb-a2e1-4e4a6c7e71d4","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:23.843Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6ca676bd-9770-4f85-9c65-b4590da6aeb9","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:23.851Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"67052bbc-fecc-40cb-a2e1-4e4a6c7e71d4","certificate":"ray-head-cert-raycluster-auth","dnsNames":["raycluster-auth-head-svc","localhost","raycluster-auth-head-svc.test-ns-qqsjt.svc","raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:24:23.864Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"67052bbc-fecc-40cb-a2e1-4e4a6c7e71d4","certificate":"ray-worker-cert-raycluster-auth","dnsNames":["raycluster-auth-worker-svc","localhost","raycluster-auth-small-group","raycluster-auth-worker-svc.test-ns-qqsjt.svc","raycluster-auth-worker-svc.test-ns-qqsjt.svc.cluster.local","*.raycluster-auth-worker-svc.test-ns-qqsjt.svc","*.raycluster-auth-worker-svc.test-ns-qqsjt.svc.cluster.local","*-worker-*.test-ns-qqsjt.svc","*-worker-*.test-ns-qqsjt.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:24:23.865Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"67052bbc-fecc-40cb-a2e1-4e4a6c7e71d4"}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4"}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4"}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4"}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4"}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:24:23.875Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:24:23.876Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:23.879Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:23.879Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:23.879Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:23.879Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:23.879Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:23.879Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4"}
{"level":"error","ts":"2026-06-10T10:24:23.879Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"90f7ef52-f9a1-4f79-b22d-dc5cdd4409e4","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a"}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a"}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","1 head service found":"raycluster-auth-head-svc"}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a"}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a"}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","mode":"IntegratedOAuth","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","cluster":"raycluster-auth","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","cluster":"raycluster-auth","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=raycluster-auth-head-svc.test-ns-qqsjt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:24:26.440Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","enabled":true}
{"level":"error","ts":"2026-06-10T10:24:26.444Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:26.444Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:24:26.444Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:26.444Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:26.444Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","filter labels":["test-ns-qqsjt",{"ray.io/cluster":"raycluster-auth","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:24:26.444Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a"}
{"level":"error","ts":"2026-06-10T10:24:26.444Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"3a1712b2-f886-4d4d-8795-f5a628d16c6a","error":"FailedCreateHeadPod\npods \"raycluster-auth-head-\" is forbidden: unable to create new content in namespace test-ns-qqsjt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:26.708Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"63466569-8ac9-4164-ad2f-59b0baed1c94","rayCluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.709Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"reconcileID":"bbfb9548-8341-401e-b1e9-c00343e9bb1c"}
{"level":"info","ts":"2026-06-10T10:24:26.709Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"68527bc8-29ad-45c6-a7fa-815ea5ea5f9c"}
{"level":"info","ts":"2026-06-10T10:24:26.708Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6a51578d-2009-4b24-ba10-2c49898a50ad","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"}}
{"level":"info","ts":"2026-06-10T10:24:26.709Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6a51578d-2009-4b24-ba10-2c49898a50ad","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.709Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6a51578d-2009-4b24-ba10-2c49898a50ad","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.709Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"63466569-8ac9-4164-ad2f-59b0baed1c94"}
{"level":"info","ts":"2026-06-10T10:24:26.709Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6a51578d-2009-4b24-ba10-2c49898a50ad","namespace":"test-ns-qqsjt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:24:26.713Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6a51578d-2009-4b24-ba10-2c49898a50ad","name":"kuberay-gateway-access","namespace":"test-ns-qqsjt"}
{"level":"info","ts":"2026-06-10T10:24:26.718Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6a51578d-2009-4b24-ba10-2c49898a50ad","name":"test-ns-qqsjt-raycluster-auth","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:24:26.722Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6a51578d-2009-4b24-ba10-2c49898a50ad","configMap":"kube-rbac-proxy-config-raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.728Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6a51578d-2009-4b24-ba10-2c49898a50ad","serviceAccount":"raycluster-auth-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:24:26.736Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.736Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"raycluster-auth","namespace":"test-ns-qqsjt"}
{"level":"info","ts":"2026-06-10T10:24:26.747Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"6a51578d-2009-4b24-ba10-2c49898a50ad","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.747Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"99653360-1486-41a9-92fa-3ff54473f2c8","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"}}
{"level":"info","ts":"2026-06-10T10:24:26.747Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"99653360-1486-41a9-92fa-3ff54473f2c8","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.747Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"99653360-1486-41a9-92fa-3ff54473f2c8","cluster":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.747Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"99653360-1486-41a9-92fa-3ff54473f2c8","namespace":"test-ns-qqsjt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:24:26.747Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"99653360-1486-41a9-92fa-3ff54473f2c8","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:24:26.747Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"11dbf948-cbd7-4acd-bce7-773ada164466","namespace":"test-ns-qqsjt","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.747Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"f17b4532-161a-4fa0-9d8c-f67f75f71816"}
{"level":"error","ts":"2026-06-10T10:24:26.751Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"99653360-1486-41a9-92fa-3ff54473f2c8","error":"rayclusters.ray.io \"raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:26.751Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"99653360-1486-41a9-92fa-3ff54473f2c8"}
{"level":"error","ts":"2026-06-10T10:24:26.751Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"99653360-1486-41a9-92fa-3ff54473f2c8","error":"rayclusters.ray.io \"raycluster-auth\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:26.751Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"5a8bcaff-3ac6-4b30-baeb-74a2f4a3b545","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"}}
{"level":"info","ts":"2026-06-10T10:24:26.751Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"5a8bcaff-3ac6-4b30-baeb-74a2f4a3b545"}
{"level":"info","ts":"2026-06-10T10:24:26.751Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"5a8bcaff-3ac6-4b30-baeb-74a2f4a3b545","namespace":"test-ns-qqsjt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:24:26.751Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"5a8bcaff-3ac6-4b30-baeb-74a2f4a3b545","namespace":"test-ns-qqsjt"}
{"level":"info","ts":"2026-06-10T10:24:26.753Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c2caa211-5570-4d25-bc45-8ed0466a9b99"}
{"level":"info","ts":"2026-06-10T10:24:26.754Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"bb56cad7-610b-4f9f-8492-4324e84424e9"}
{"level":"info","ts":"2026-06-10T10:24:26.754Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c5f9b691-a421-4574-ba86-8c3eee9f0362","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"}}
{"level":"info","ts":"2026-06-10T10:24:26.755Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c5f9b691-a421-4574-ba86-8c3eee9f0362"}
{"level":"info","ts":"2026-06-10T10:24:26.755Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c5f9b691-a421-4574-ba86-8c3eee9f0362","namespace":"test-ns-qqsjt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:24:26.755Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"c5f9b691-a421-4574-ba86-8c3eee9f0362","namespace":"test-ns-qqsjt"}
{"level":"info","ts":"2026-06-10T10:24:26.799Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"787b851b-59db-4d47-9f9a-88d880f1642d","namespacedName":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"}}
{"level":"info","ts":"2026-06-10T10:24:26.799Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"787b851b-59db-4d47-9f9a-88d880f1642d"}
{"level":"info","ts":"2026-06-10T10:24:26.799Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"787b851b-59db-4d47-9f9a-88d880f1642d","namespace":"test-ns-qqsjt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:24:26.799Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"787b851b-59db-4d47-9f9a-88d880f1642d","namespace":"test-ns-qqsjt"}
{"level":"error","ts":"2026-06-10T10:24:26.802Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"11dbf948-cbd7-4acd-bce7-773ada164466","certificate":"ray-worker-cert-raycluster-auth","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:26.819Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"11dbf948-cbd7-4acd-bce7-773ada164466","errorCount":1}
{"level":"info","ts":"2026-06-10T10:24:26.819Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"11dbf948-cbd7-4acd-bce7-773ada164466"}
{"level":"error","ts":"2026-06-10T10:24:26.819Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"11dbf948-cbd7-4acd-bce7-773ada164466","error":"certificates.cert-manager.io \"ray-worker-cert-raycluster-auth\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:24:26.824Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"2a0e3871-e483-4fae-91dc-636286a34f44","namespace":"test-ns-qqsjt","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:26.824Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth","namespace":"test-ns-qqsjt"},"namespace":"test-ns-qqsjt","name":"raycluster-auth","reconcileID":"2a0e3871-e483-4fae-91dc-636286a34f44","namespace":"test-ns-qqsjt","clusterName":"raycluster-auth"}
{"level":"info","ts":"2026-06-10T10:24:46.900Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"cf496f6c-f617-4df5-b531-6b5b35d657b9","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:24:46.900Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"cf496f6c-f617-4df5-b531-6b5b35d657b9","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:24:46.900Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"cf496f6c-f617-4df5-b531-6b5b35d657b9","rayCluster":"raycluster","podIPs":["10.133.0.13"]}
{"level":"info","ts":"2026-06-10T10:25:35.407Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd"}
{"level":"info","ts":"2026-06-10T10:25:35.407Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd"}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","podName":"raycluster-head-jjvtb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:25:35.408Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"c5f6683e-7454-4d93-b315-13143c0815dd","seconds":300}
{"level":"info","ts":"2026-06-10T10:25:46.901Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"27cd0b6f-49a6-4ecf-88ab-c61c5420c191","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:25:46.902Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"27cd0b6f-49a6-4ecf-88ab-c61c5420c191","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:25:46.902Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"27cd0b6f-49a6-4ecf-88ab-c61c5420c191","rayCluster":"raycluster","podIPs":["10.133.0.13"]}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97"}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97"}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","podName":"raycluster-head-jjvtb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:24:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","podName":"raycluster-head-jjvtb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:24:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:25:48.481Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9b3e3702-8a3d-4684-a786-777a75b60f97","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:24:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:25:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:25:48.494Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6dcf25c7-4431-4cd0-9cbb-9624f5e8f8fa","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:25:48.494Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"c3408b4a-96e0-447b-b606-dc09de2f62dd","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:25:48.494Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"c3408b4a-96e0-447b-b606-dc09de2f62dd","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:25:48.494Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"c3408b4a-96e0-447b-b606-dc09de2f62dd","rayCluster":"raycluster","podIPs":["10.133.0.13"]}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805"}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805"}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:25:48.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:25:48.506Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:25:48.506Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:25:48.506Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:25:48.506Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f1123805-4096-4713-9293-f66c8c4a4805","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:25:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:25:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:25:48.510Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6dcf25c7-4431-4cd0-9cbb-9624f5e8f8fa","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:25:48.517Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"50e3ca78-0dea-4a2e-8e54-e54d81ab9e5a","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:25:48.517Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"50e3ca78-0dea-4a2e-8e54-e54d81ab9e5a","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:25:48.517Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"50e3ca78-0dea-4a2e-8e54-e54d81ab9e5a","rayCluster":"raycluster","podIPs":["10.133.0.13"]}
{"level":"info","ts":"2026-06-10T10:25:48.521Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6dcf25c7-4431-4cd0-9cbb-9624f5e8f8fa","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:25:48.521Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"6dcf25c7-4431-4cd0-9cbb-9624f5e8f8fa","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:25:48.521Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f1b7a9b9-51c7-4a67-aa4f-b090e69fc60f","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:25:48.533Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f1b7a9b9-51c7-4a67-aa4f-b090e69fc60f","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:25:48.545Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f1b7a9b9-51c7-4a67-aa4f-b090e69fc60f","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:25:48.545Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f1b7a9b9-51c7-4a67-aa4f-b090e69fc60f","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:25:50.494Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca"}
{"level":"info","ts":"2026-06-10T10:25:50.494Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca"}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:25:50.495Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"7965a0e0-b87f-446f-aa7f-519967dd9eca","seconds":300}
{"level":"info","ts":"2026-06-10T10:26:46.902Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"5677f802-6c72-4c91-9bff-6b6373156452","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:26:46.903Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"5677f802-6c72-4c91-9bff-6b6373156452","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:26:46.903Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"5677f802-6c72-4c91-9bff-6b6373156452","rayCluster":"raycluster","podIPs":["10.133.0.13"]}
{"level":"info","ts":"2026-06-10T10:27:46.904Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"df4fb092-1d37-449e-a74d-0dc75ee8f0d1","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:27:46.904Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"df4fb092-1d37-449e-a74d-0dc75ee8f0d1","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:27:46.904Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"df4fb092-1d37-449e-a74d-0dc75ee8f0d1","rayCluster":"raycluster","podIPs":["10.133.0.13"]}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe"}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe"}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:31.202Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:31.203Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:28:31.203Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"976d8a29-7140-4245-b54b-64006c942bfe","seconds":300}
{"level":"info","ts":"2026-06-10T10:28:39.237Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb"}
{"level":"info","ts":"2026-06-10T10:28:39.237Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb"}
{"level":"info","ts":"2026-06-10T10:28:39.237Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:39.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:39.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:39.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:28:39.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:39.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:39.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:28:39.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:39.238Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:39.238Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","podName":"raycluster-small-group-worker-7j6bp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:39.238Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:28:39.238Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"f0b1ac5f-41f3-47e1-af19-1927f873dfeb","seconds":300}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2"}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2"}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","podName":"raycluster-small-group-worker-7j6bp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:24:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","podName":"raycluster-small-group-worker-7j6bp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:24:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:28:40.247Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"310cea56-e348-4306-bf11-85e142841dc2","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:25:48Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:40.262Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4d5c18ba-f1ee-43fe-8056-c2b506df9c7a","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:28:40.262Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"799ca1d7-603c-4dae-aff4-886a30ac2506","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:28:40.262Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4d5c18ba-f1ee-43fe-8056-c2b506df9c7a","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:28:40.262Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4d5c18ba-f1ee-43fe-8056-c2b506df9c7a","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:28:40.286Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"799ca1d7-603c-4dae-aff4-886a30ac2506","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:28:40.297Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"799ca1d7-603c-4dae-aff4-886a30ac2506","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:28:40.297Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"799ca1d7-603c-4dae-aff4-886a30ac2506","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:28:40.305Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4d5c18ba-f1ee-43fe-8056-c2b506df9c7a","certificate":"ray-head-cert-raycluster","dnsNames":["raycluster-head-svc","localhost","raycluster-head-svc.test-ns-ccmjv.svc","raycluster-head-svc.test-ns-ccmjv.svc.cluster.local"],"ipAddresses":["10.133.0.13","10.132.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:28:40.333Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4d5c18ba-f1ee-43fe-8056-c2b506df9c7a","certificate":"ray-worker-cert-raycluster","dnsNames":["raycluster-worker-svc","localhost","raycluster-small-group","raycluster-worker-svc.test-ns-ccmjv.svc","raycluster-worker-svc.test-ns-ccmjv.svc.cluster.local","*.raycluster-worker-svc.test-ns-ccmjv.svc","*.raycluster-worker-svc.test-ns-ccmjv.svc.cluster.local","*-worker-*.test-ns-ccmjv.svc","*-worker-*.test-ns-ccmjv.svc.cluster.local"],"ipAddresses":["10.133.0.13","10.132.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:28:40.333Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4d5c18ba-f1ee-43fe-8056-c2b506df9c7a"}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8"}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8"}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","podName":"raycluster-small-group-worker-7j6bp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:24:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","podName":"raycluster-small-group-worker-7j6bp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:24:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:28:42.263Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"1159b48b-d59d-4485-b27a-49352ca08cc8","seconds":300}
{"level":"info","ts":"2026-06-10T10:28:46.904Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"2edc95dc-22f1-41ed-9294-ad8ec8ff0b94","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:28:46.904Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"2edc95dc-22f1-41ed-9294-ad8ec8ff0b94","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:28:46.904Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"2edc95dc-22f1-41ed-9294-ad8ec8ff0b94","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:28:46.904Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"2edc95dc-22f1-41ed-9294-ad8ec8ff0b94","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53"}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53"}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:51.267Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:28:51.268Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"48dc51f1-7ebf-450c-bbb9-f66cf9247e53","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:51.279Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"0ce81f95-cf2b-4e4b-8b5c-26689ffb67c4","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:28:51.279Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4a4e6685-170b-4bc0-9a33-f6f401d49121","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:28:51.279Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4a4e6685-170b-4bc0-9a33-f6f401d49121","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:28:51.279Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4a4e6685-170b-4bc0-9a33-f6f401d49121","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:28:51.279Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"4a4e6685-170b-4bc0-9a33-f6f401d49121","rayCluster":"raycluster","podIPs":["10.132.0.41","10.133.0.13"]}
{"level":"info","ts":"2026-06-10T10:28:51.294Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"0ce81f95-cf2b-4e4b-8b5c-26689ffb67c4","name":"raycluster-head"}
{"level":"info","ts":"2026-06-10T10:28:51.304Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"0ce81f95-cf2b-4e4b-8b5c-26689ffb67c4","name":"raycluster-workers"}
{"level":"info","ts":"2026-06-10T10:28:51.304Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"0ce81f95-cf2b-4e4b-8b5c-26689ffb67c4","cluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:28:51.367Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c8deeab3-dbfc-4bfd-a734-8d13395ede0b","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:51.367Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c8deeab3-dbfc-4bfd-a734-8d13395ede0b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:28:51.378Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c8deeab3-dbfc-4bfd-a734-8d13395ede0b"}
{"level":"info","ts":"2026-06-10T10:28:51.378Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c8deeab3-dbfc-4bfd-a734-8d13395ede0b","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:28:51.378Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c8deeab3-dbfc-4bfd-a734-8d13395ede0b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-v546s","rayClusterName":"counter-6d72m","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:28:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:28:51.378Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c8deeab3-dbfc-4bfd-a734-8d13395ede0b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:28:51.388Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f4240f10-5311-49e9-9b75-3374f8571283","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:51.388Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f4240f10-5311-49e9-9b75-3374f8571283"}
{"level":"info","ts":"2026-06-10T10:28:51.388Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f4240f10-5311-49e9-9b75-3374f8571283","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:28:51.388Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f4240f10-5311-49e9-9b75-3374f8571283","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-h6n84","rayClusterName":"counter-wcpmw","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:28:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:28:51.388Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f4240f10-5311-49e9-9b75-3374f8571283","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:28:51.394Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f4240f10-5311-49e9-9b75-3374f8571283","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-10T10:28:51.394Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f4240f10-5311-49e9-9b75-3374f8571283"}
{"level":"error","ts":"2026-06-10T10:28:51.394Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f4240f10-5311-49e9-9b75-3374f8571283","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.394Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"0929b0a3-32fa-4b56-8e13-1d14020400d4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:51.394Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"0929b0a3-32fa-4b56-8e13-1d14020400d4","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.394Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"0929b0a3-32fa-4b56-8e13-1d14020400d4","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.394Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"0929b0a3-32fa-4b56-8e13-1d14020400d4","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.412Z","logger":"raycluster-resource","msg":"default","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.412Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-6d72m","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:28:51.419Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"0929b0a3-32fa-4b56-8e13-1d14020400d4","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"0929b0a3-32fa-4b56-8e13-1d14020400d4"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"0929b0a3-32fa-4b56-8e13-1d14020400d4","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"45d34915-905c-4cac-ab47-9be3b5578ad3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"45d34915-905c-4cac-ab47-9be3b5578ad3","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"45d34915-905c-4cac-ab47-9be3b5578ad3","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"45d34915-905c-4cac-ab47-9be3b5578ad3","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","namespacedName":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","mode":"IntegratedOAuth","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.420Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"e394ece3-a326-40d7-95b5-28cecfc5ecf5","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.424Z","logger":"raycluster-resource","msg":"default","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.424Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-6d72m","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:28:51.427Z","logger":"raycluster-resource","msg":"default","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.427Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-6d72m","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:28:51.427Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","name":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:51.427Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","error":"head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"error","ts":"2026-06-10T10:28:51.427Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.427Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:51.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.428Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.428Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-6d72m-head-svc"}}
{"level":"info","ts":"2026-06-10T10:28:51.428Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","name":"counter-6d72m","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"e394ece3-a326-40d7-95b5-28cecfc5ecf5","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","name":"counter-6d72m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"counter-6d72m"}}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8"}
{"level":"error","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"8f918821-1f6d-4c78-927a-44f322d472f8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4b5997c5-5032-48bb-8f6b-d0be0f3c27c8","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","error":"head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"error","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-6d72m-head-svc"}}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"45d34915-905c-4cac-ab47-9be3b5578ad3"}
{"level":"error","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"45d34915-905c-4cac-ab47-9be3b5578ad3","error":"rayclusters.ray.io \"counter-6d72m\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.499Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","name":"counter-6d72m","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:51.505Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","name":"counter-6d72m-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","name":"counter-6d72m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"counter-6d72m"}}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5"}
{"level":"error","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"b4526756-d9e9-4663-9d89-be6c9927c6c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652"}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652"}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","error":"head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"error","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-6d72m-head-svc"}}
{"level":"info","ts":"2026-06-10T10:28:51.506Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","name":"counter-6d72m","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:51.511Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655"}
{"level":"info","ts":"2026-06-10T10:28:51.511Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655","rayCluster":"counter-6d72m","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:28:51.511Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.512Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4b5997c5-5032-48bb-8f6b-d0be0f3c27c8","name":"counter-6d72m-head"}
{"level":"info","ts":"2026-06-10T10:28:51.599Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","name":"kuberay-gateway-access","namespace":"test-ns-bbzgk","operation":"created"}
{"level":"info","ts":"2026-06-10T10:28:51.599Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6588acb2-81a6-4554-9c14-d7eefba5d010","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6588acb2-81a6-4554-9c14-d7eefba5d010","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6588acb2-81a6-4554-9c14-d7eefba5d010","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652"}
{"level":"error","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"41aaf117-de2f-423a-9c3a-8bf53e9ea652","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6588acb2-81a6-4554-9c14-d7eefba5d010"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6588acb2-81a6-4554-9c14-d7eefba5d010","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6288d959-9323-48b4-935d-e47a0bca21d0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6288d959-9323-48b4-935d-e47a0bca21d0","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6288d959-9323-48b4-935d-e47a0bca21d0","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","error":"head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6288d959-9323-48b4-935d-e47a0bca21d0"}
{"level":"error","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6288d959-9323-48b4-935d-e47a0bca21d0","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","oldEndpoints":null,"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{},"newHead":{"serviceName":"counter-6d72m-head-svc"}}
{"level":"info","ts":"2026-06-10T10:28:51.600Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","name":"counter-6d72m","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:51.603Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4b5997c5-5032-48bb-8f6b-d0be0f3c27c8","name":"counter-6d72m-workers"}
{"level":"info","ts":"2026-06-10T10:28:51.603Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4b5997c5-5032-48bb-8f6b-d0be0f3c27c8","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.603Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"2323120d-0594-41fb-8ebd-f1c0a5bd113c","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.606Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","name":"counter-6d72m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"counter-6d72m"}}
{"level":"info","ts":"2026-06-10T10:28:51.606Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd"}
{"level":"error","ts":"2026-06-10T10:28:51.606Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c0bbde9b-5d3f-4c7b-9f67-42167e7025dd","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.607Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","name":"test-ns-bbzgk-counter-6d72m","operation":"created"}
{"level":"info","ts":"2026-06-10T10:28:51.610Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655"}
{"level":"info","ts":"2026-06-10T10:28:51.611Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655","rayCluster":"counter-6d72m","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:28:51.611Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","name":"kube-rbac-proxy-config-counter-6d72m","operation":"created"}
{"level":"info","ts":"2026-06-10T10:28:51.615Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"2323120d-0594-41fb-8ebd-f1c0a5bd113c","name":"counter-6d72m-head"}
{"level":"error","ts":"2026-06-10T10:28:51.617Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.617Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598"}
{"level":"error","ts":"2026-06-10T10:28:51.617Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"dfc70d27-abfa-4d0a-a0a9-f19ed4957598","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.617Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"9389b937-ee1e-44df-b52a-27770d2d90c2","namespacedName":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.617Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"9389b937-ee1e-44df-b52a-27770d2d90c2","mode":"IntegratedOAuth","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.617Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"9389b937-ee1e-44df-b52a-27770d2d90c2","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"cc3eba3b-ac72-4f49-a347-326994e1a16b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"cc3eba3b-ac72-4f49-a347-326994e1a16b","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"cc3eba3b-ac72-4f49-a347-326994e1a16b","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055","error":"head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"error","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-06-10T10:28:51.699Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"9389b937-ee1e-44df-b52a-27770d2d90c2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"9389b937-ee1e-44df-b52a-27770d2d90c2"}
{"level":"error","ts":"2026-06-10T10:28:51.699Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"9389b937-ee1e-44df-b52a-27770d2d90c2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"b5cfa6dc-c5ce-4200-9657-fa6d8ba118e5","namespacedName":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"cc3eba3b-ac72-4f49-a347-326994e1a16b"}
{"level":"info","ts":"2026-06-10T10:28:51.700Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"cc3eba3b-ac72-4f49-a347-326994e1a16b","RayCluster":"counter-6d72m","State":""}
{"level":"error","ts":"2026-06-10T10:28:51.699Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bc41ddbd-a4a2-4021-9ffc-3e804edec055","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.700Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"b5cfa6dc-c5ce-4200-9657-fa6d8ba118e5","mode":"IntegratedOAuth","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.699Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655"}
{"level":"info","ts":"2026-06-10T10:28:51.700Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"b5cfa6dc-c5ce-4200-9657-fa6d8ba118e5","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.700Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655","rayCluster":"counter-6d72m","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:28:51.700Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d842e2e-e6e8-4650-b1e3-f6c148dc2655"}
{"level":"info","ts":"2026-06-10T10:28:51.700Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"ccba5e0b-c526-4c6d-98d7-59fbbfe0801b","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.700Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"ccba5e0b-c526-4c6d-98d7-59fbbfe0801b"}
{"level":"info","ts":"2026-06-10T10:28:51.704Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"2323120d-0594-41fb-8ebd-f1c0a5bd113c","name":"counter-6d72m-workers"}
{"level":"info","ts":"2026-06-10T10:28:51.704Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"2323120d-0594-41fb-8ebd-f1c0a5bd113c","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.704Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"89771281-e607-4e2a-aff0-b355931cad16","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"b5cfa6dc-c5ce-4200-9657-fa6d8ba118e5","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d708b8b-7573-4292-99c6-fb865655ef7d","namespacedName":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d708b8b-7573-4292-99c6-fb865655ef7d","mode":"IntegratedOAuth","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d708b8b-7573-4292-99c6-fb865655ef7d","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"3e1ce3fd-613f-4703-8783-62ebb7e2b0c9","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"3e1ce3fd-613f-4703-8783-62ebb7e2b0c9"}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"04d5ad17-8526-4057-b61a-1c5d69159421","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"04d5ad17-8526-4057-b61a-1c5d69159421","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"04d5ad17-8526-4057-b61a-1c5d69159421","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"04d5ad17-8526-4057-b61a-1c5d69159421"}
{"level":"info","ts":"2026-06-10T10:28:51.711Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"04d5ad17-8526-4057-b61a-1c5d69159421","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:28:51.717Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"89771281-e607-4e2a-aff0-b355931cad16","name":"counter-6d72m-head"}
{"level":"error","ts":"2026-06-10T10:28:51.717Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d708b8b-7573-4292-99c6-fb865655ef7d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.718Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d708b8b-7573-4292-99c6-fb865655ef7d"}
{"level":"error","ts":"2026-06-10T10:28:51.718Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5d708b8b-7573-4292-99c6-fb865655ef7d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.723Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"a5a09296-c41d-4226-a3fa-4f6b7123f41c","namespacedName":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:51.723Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"a5a09296-c41d-4226-a3fa-4f6b7123f41c","mode":"IntegratedOAuth","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.723Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"a5a09296-c41d-4226-a3fa-4f6b7123f41c","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.728Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"89771281-e607-4e2a-aff0-b355931cad16","name":"counter-6d72m-workers"}
{"level":"info","ts":"2026-06-10T10:28:51.728Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"89771281-e607-4e2a-aff0-b355931cad16","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.728Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"ce201b93-1443-42dd-a591-67f0cd41825e","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f"}
{"level":"info","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f"}
{"level":"info","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f","error":"head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"error","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f"}
{"level":"error","ts":"2026-06-10T10:28:51.799Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"6355032b-bb2e-42ed-9ecf-f37e0f74c17f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.806Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"ce201b93-1443-42dd-a591-67f0cd41825e","name":"counter-6d72m-head"}
{"level":"info","ts":"2026-06-10T10:28:51.811Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"a5a09296-c41d-4226-a3fa-4f6b7123f41c","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.815Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"ce201b93-1443-42dd-a591-67f0cd41825e","name":"counter-6d72m-workers"}
{"level":"info","ts":"2026-06-10T10:28:51.815Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"ce201b93-1443-42dd-a591-67f0cd41825e","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7"}
{"level":"info","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7"}
{"level":"info","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7","error":"head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"error","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7"}
{"level":"error","ts":"2026-06-10T10:28:51.960Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4554ba65-27a1-4ae5-9533-244a6525c7f7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1"}
{"level":"info","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1"}
{"level":"info","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1","error":"head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"error","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1"}
{"level":"error","ts":"2026-06-10T10:28:52.280Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"bf403080-2f7d-451f-9028-97a93a555fb1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:52.921Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301"}
{"level":"info","ts":"2026-06-10T10:28:52.921Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301"}
{"level":"info","ts":"2026-06-10T10:28:52.921Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:52.921Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301","error":"head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"error","ts":"2026-06-10T10:28:52.921Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:52.921Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found"}
{"level":"info","ts":"2026-06-10T10:28:52.921Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:52.921Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:52.921Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"counter-6d72m","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:28:52.922Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301"}
{"level":"error","ts":"2026-06-10T10:28:52.922Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"310470a9-e65c-411f-b99e-b651df56a301","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-6d72m not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb"}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb"}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:28:53.279Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"9ac328f0-8c40-4015-a79f-03bdd2aac5cb","seconds":300}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d"}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d"}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d"}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d"}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","mode":"IntegratedOAuth","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","cluster":"counter-6d72m","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","cluster":"counter-6d72m","authType":"IntegratedOAuth","serviceAccount":"counter-6d72m-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-6d72m","ray.io/group":"headgroup","ray.io/identifier":"counter-6d72m-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local","num-cpus":"4","num-gpus":"4","resources":"'{\"R1\": 4}'"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=4  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-06-10T10:28:54.203Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","enabled":true}
{"level":"info","ts":"2026-06-10T10:28:54.226Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","name":"counter-6d72m-head-njqmv"}
{"level":"info","ts":"2026-06-10T10:28:54.226Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.226Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.226Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-6d72m-head-svc"},"newHead":{"podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"}}
{"level":"info","ts":"2026-06-10T10:28:54.226Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"74d4b54e-e773-48bd-89f7-145273cb5b9d","name":"counter-6d72m","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5"}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5"}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-6d72m-head-svc"},"newHead":{"podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"}}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5","name":"counter-6d72m","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:54.239Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"a05779ee-1500-4bab-8485-0d01a5001268","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.240Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"0c8f5505-4276-443d-9a71-b2605e2c0e36","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c381a311-0999-42a0-b275-da876063aa6e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:54.240Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c381a311-0999-42a0-b275-da876063aa6e","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.240Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c381a311-0999-42a0-b275-da876063aa6e","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:54.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c381a311-0999-42a0-b275-da876063aa6e"}
{"level":"info","ts":"2026-06-10T10:28:54.240Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c381a311-0999-42a0-b275-da876063aa6e","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5","name":"counter-6d72m","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"counter-6d72m"}}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5"}
{"level":"error","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"db464e6d-73cb-437c-ad4f-9bfd164b3da5","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-6d72m\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"d0dfd317-0ac1-49ad-b8d3-da7624d4bc92"}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"d0dfd317-0ac1-49ad-b8d3-da7624d4bc92"}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"d0dfd317-0ac1-49ad-b8d3-da7624d4bc92","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"d0dfd317-0ac1-49ad-b8d3-da7624d4bc92","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"d0dfd317-0ac1-49ad-b8d3-da7624d4bc92","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"d0dfd317-0ac1-49ad-b8d3-da7624d4bc92","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"d0dfd317-0ac1-49ad-b8d3-da7624d4bc92","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"d0dfd317-0ac1-49ad-b8d3-da7624d4bc92","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:28:54.248Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"d0dfd317-0ac1-49ad-b8d3-da7624d4bc92","seconds":300}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7664c813-3162-4cd9-8759-9dd46dd2cac3"}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7664c813-3162-4cd9-8759-9dd46dd2cac3"}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7664c813-3162-4cd9-8759-9dd46dd2cac3","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7664c813-3162-4cd9-8759-9dd46dd2cac3","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7664c813-3162-4cd9-8759-9dd46dd2cac3","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7664c813-3162-4cd9-8759-9dd46dd2cac3","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7664c813-3162-4cd9-8759-9dd46dd2cac3","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7664c813-3162-4cd9-8759-9dd46dd2cac3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7664c813-3162-4cd9-8759-9dd46dd2cac3","seconds":300}
{"level":"info","ts":"2026-06-10T10:28:54.254Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"0c8f5505-4276-443d-9a71-b2605e2c0e36","name":"counter-6d72m-head"}
{"level":"info","ts":"2026-06-10T10:28:54.266Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"0c8f5505-4276-443d-9a71-b2605e2c0e36","name":"counter-6d72m-workers"}
{"level":"info","ts":"2026-06-10T10:28:54.266Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"0c8f5505-4276-443d-9a71-b2605e2c0e36","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.267Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e31f3430-fa24-4391-996c-0eb65dc17c34"}
{"level":"info","ts":"2026-06-10T10:28:54.267Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e31f3430-fa24-4391-996c-0eb65dc17c34"}
{"level":"info","ts":"2026-06-10T10:28:54.267Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e31f3430-fa24-4391-996c-0eb65dc17c34","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:54.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e31f3430-fa24-4391-996c-0eb65dc17c34","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:54.267Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e31f3430-fa24-4391-996c-0eb65dc17c34","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:54.267Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e31f3430-fa24-4391-996c-0eb65dc17c34","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.267Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e31f3430-fa24-4391-996c-0eb65dc17c34","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.267Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e31f3430-fa24-4391-996c-0eb65dc17c34","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:28:54.267Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e31f3430-fa24-4391-996c-0eb65dc17c34","name":"counter-6d72m","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:54.299Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"a05779ee-1500-4bab-8485-0d01a5001268","rayCluster":"counter-6d72m","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:28:54.299Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"45e0aadc-b8f0-433c-9bcc-ab13f362c8c5","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.299Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5aa8817c-2990-4586-9ba2-5431915681eb","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.299Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"5aa8817c-2990-4586-9ba2-5431915681eb","rayCluster":"counter-6d72m","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:28:54.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"79c7e89e-e9d9-421a-8cec-ca53d5ccf924","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:54.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"79c7e89e-e9d9-421a-8cec-ca53d5ccf924","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"79c7e89e-e9d9-421a-8cec-ca53d5ccf924","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:54.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"79c7e89e-e9d9-421a-8cec-ca53d5ccf924"}
{"level":"info","ts":"2026-06-10T10:28:54.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"79c7e89e-e9d9-421a-8cec-ca53d5ccf924","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:28:54.319Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"45e0aadc-b8f0-433c-9bcc-ab13f362c8c5","name":"counter-6d72m-head"}
{"level":"info","ts":"2026-06-10T10:28:54.345Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"45e0aadc-b8f0-433c-9bcc-ab13f362c8c5","name":"counter-6d72m-workers"}
{"level":"info","ts":"2026-06-10T10:28:54.345Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"45e0aadc-b8f0-433c-9bcc-ab13f362c8c5","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.601Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"9398f4a7-6fb0-4385-aa08-1eb12fda5d60","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:54.601Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"9398f4a7-6fb0-4385-aa08-1eb12fda5d60","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:54.601Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"9398f4a7-6fb0-4385-aa08-1eb12fda5d60","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:54.601Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"9398f4a7-6fb0-4385-aa08-1eb12fda5d60"}
{"level":"info","ts":"2026-06-10T10:28:54.601Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"9398f4a7-6fb0-4385-aa08-1eb12fda5d60","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:28:54.674Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c4f88afe-a757-4e66-943b-25aa0b777484"}
{"level":"info","ts":"2026-06-10T10:28:54.674Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c4f88afe-a757-4e66-943b-25aa0b777484"}
{"level":"info","ts":"2026-06-10T10:28:54.674Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c4f88afe-a757-4e66-943b-25aa0b777484","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:54.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c4f88afe-a757-4e66-943b-25aa0b777484","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:54.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c4f88afe-a757-4e66-943b-25aa0b777484","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:54.674Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c4f88afe-a757-4e66-943b-25aa0b777484","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.674Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c4f88afe-a757-4e66-943b-25aa0b777484","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:54.675Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c4f88afe-a757-4e66-943b-25aa0b777484","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:28:54.675Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"c4f88afe-a757-4e66-943b-25aa0b777484","seconds":300}
{"level":"info","ts":"2026-06-10T10:28:55.709Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"61a9e36a-5885-4dce-836b-0364bbecb405"}
{"level":"info","ts":"2026-06-10T10:28:55.709Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"61a9e36a-5885-4dce-836b-0364bbecb405"}
{"level":"info","ts":"2026-06-10T10:28:55.709Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"61a9e36a-5885-4dce-836b-0364bbecb405","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:55.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"61a9e36a-5885-4dce-836b-0364bbecb405","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:55.709Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"61a9e36a-5885-4dce-836b-0364bbecb405","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:55.709Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"61a9e36a-5885-4dce-836b-0364bbecb405","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:55.709Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"61a9e36a-5885-4dce-836b-0364bbecb405","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:55.709Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"61a9e36a-5885-4dce-836b-0364bbecb405","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"newHead":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"}}
{"level":"info","ts":"2026-06-10T10:28:55.710Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"61a9e36a-5885-4dce-836b-0364bbecb405","name":"counter-6d72m","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:28:55.721Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"cca02f64-a2df-470a-b0da-97f8a402b086","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:55.721Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4057aa83-8eb4-4d10-a98c-daef2f856d2e","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:55.721Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"cca02f64-a2df-470a-b0da-97f8a402b086","pod":"counter-6d72m-head-njqmv","ip":"10.134.0.16"}
{"level":"info","ts":"2026-06-10T10:28:55.721Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"4285c4dd-1dbe-44ec-89a6-85d53ac38a71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:55.721Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"4285c4dd-1dbe-44ec-89a6-85d53ac38a71","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:55.721Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"4285c4dd-1dbe-44ec-89a6-85d53ac38a71","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:55.721Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"4285c4dd-1dbe-44ec-89a6-85d53ac38a71"}
{"level":"info","ts":"2026-06-10T10:28:55.721Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"4285c4dd-1dbe-44ec-89a6-85d53ac38a71","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:28:55.732Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4057aa83-8eb4-4d10-a98c-daef2f856d2e","name":"counter-6d72m-head"}
{"level":"info","ts":"2026-06-10T10:28:55.732Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"cca02f64-a2df-470a-b0da-97f8a402b086","certificate":"ray-head-cert-counter-6d72m","dnsNames":["counter-6d72m-head-svc","localhost","counter-6d72m-head-svc.test-ns-bbzgk.svc","counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.16","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:28:55.740Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4057aa83-8eb4-4d10-a98c-daef2f856d2e","name":"counter-6d72m-workers"}
{"level":"info","ts":"2026-06-10T10:28:55.740Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4057aa83-8eb4-4d10-a98c-daef2f856d2e","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:55.743Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"cca02f64-a2df-470a-b0da-97f8a402b086","certificate":"ray-worker-cert-counter-6d72m","dnsNames":["counter-6d72m-worker-svc","localhost","counter-6d72m-worker-svc.test-ns-bbzgk.svc","counter-6d72m-worker-svc.test-ns-bbzgk.svc.cluster.local","*.counter-6d72m-worker-svc.test-ns-bbzgk.svc","*.counter-6d72m-worker-svc.test-ns-bbzgk.svc.cluster.local","*-worker-*.test-ns-bbzgk.svc","*-worker-*.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.16","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:28:55.744Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"cca02f64-a2df-470a-b0da-97f8a402b086","rayCluster":"counter-6d72m","podIPs":["10.134.0.16"]}
{"level":"info","ts":"2026-06-10T10:28:56.300Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"25f5705e-94c8-4631-839f-f6b8b36b87c1"}
{"level":"info","ts":"2026-06-10T10:28:56.300Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"25f5705e-94c8-4631-839f-f6b8b36b87c1"}
{"level":"info","ts":"2026-06-10T10:28:56.300Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"25f5705e-94c8-4631-839f-f6b8b36b87c1","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:28:56.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"25f5705e-94c8-4631-839f-f6b8b36b87c1","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:28:56.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"25f5705e-94c8-4631-839f-f6b8b36b87c1","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:28:56.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"25f5705e-94c8-4631-839f-f6b8b36b87c1","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:56.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"25f5705e-94c8-4631-839f-f6b8b36b87c1","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:28:56.300Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"25f5705e-94c8-4631-839f-f6b8b36b87c1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:28:56.300Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"25f5705e-94c8-4631-839f-f6b8b36b87c1","seconds":300}
{"level":"info","ts":"2026-06-10T10:28:57.601Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"a7820a3b-2e2b-4b03-934a-ec5197e8f23e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:28:57.601Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"a7820a3b-2e2b-4b03-934a-ec5197e8f23e","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:28:57.601Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"a7820a3b-2e2b-4b03-934a-ec5197e8f23e","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:28:57.601Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"a7820a3b-2e2b-4b03-934a-ec5197e8f23e"}
{"level":"info","ts":"2026-06-10T10:28:57.601Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"a7820a3b-2e2b-4b03-934a-ec5197e8f23e","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:29:00.602Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c9da710e-bc72-4de9-86d3-1148c9cf8736","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:00.603Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c9da710e-bc72-4de9-86d3-1148c9cf8736","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:00.603Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c9da710e-bc72-4de9-86d3-1148c9cf8736","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:00.603Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c9da710e-bc72-4de9-86d3-1148c9cf8736"}
{"level":"info","ts":"2026-06-10T10:29:00.603Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"c9da710e-bc72-4de9-86d3-1148c9cf8736","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:29:00.734Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"2231556b-a0ea-40d2-a4ed-e7ef9443dca2"}
{"level":"info","ts":"2026-06-10T10:29:00.734Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"2231556b-a0ea-40d2-a4ed-e7ef9443dca2"}
{"level":"info","ts":"2026-06-10T10:29:00.734Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"2231556b-a0ea-40d2-a4ed-e7ef9443dca2","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:00.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"2231556b-a0ea-40d2-a4ed-e7ef9443dca2","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:00.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"2231556b-a0ea-40d2-a4ed-e7ef9443dca2","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:00.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"2231556b-a0ea-40d2-a4ed-e7ef9443dca2","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:00.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"2231556b-a0ea-40d2-a4ed-e7ef9443dca2","podName":"counter-6d72m-head-njqmv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:00.734Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"2231556b-a0ea-40d2-a4ed-e7ef9443dca2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:29:00.734Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"2231556b-a0ea-40d2-a4ed-e7ef9443dca2","seconds":300}
{"level":"info","ts":"2026-06-10T10:29:01.739Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7629343f-e459-4f67-91d5-3006fc40a144"}
{"level":"info","ts":"2026-06-10T10:29:01.739Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7629343f-e459-4f67-91d5-3006fc40a144"}
{"level":"info","ts":"2026-06-10T10:29:01.739Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7629343f-e459-4f67-91d5-3006fc40a144","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:01.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7629343f-e459-4f67-91d5-3006fc40a144","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:01.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7629343f-e459-4f67-91d5-3006fc40a144","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:01.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7629343f-e459-4f67-91d5-3006fc40a144","podName":"counter-6d72m-head-njqmv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:28:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:29:01.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7629343f-e459-4f67-91d5-3006fc40a144","podName":"counter-6d72m-head-njqmv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:28:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:29:01.739Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7629343f-e459-4f67-91d5-3006fc40a144","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:29:01.739Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"7629343f-e459-4f67-91d5-3006fc40a144","name":"counter-6d72m","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:29:01.752Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"2e8447b9-5eed-4cc1-b747-a7848e3f4de4","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:01.752Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"30d18b77-c211-405d-8acb-1915676e0b21","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:01.752Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"2e8447b9-5eed-4cc1-b747-a7848e3f4de4","pod":"counter-6d72m-head-njqmv","ip":"10.134.0.16"}
{"level":"info","ts":"2026-06-10T10:29:01.752Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"2e8447b9-5eed-4cc1-b747-a7848e3f4de4","rayCluster":"counter-6d72m","podIPs":["10.134.0.16"]}
{"level":"info","ts":"2026-06-10T10:29:01.752Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"327bd27a-398d-4cb7-a89d-a2e875de5b4d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:01.752Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"327bd27a-398d-4cb7-a89d-a2e875de5b4d","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:01.752Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"327bd27a-398d-4cb7-a89d-a2e875de5b4d","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:01.753Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"327bd27a-398d-4cb7-a89d-a2e875de5b4d"}
{"level":"info","ts":"2026-06-10T10:29:01.753Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"327bd27a-398d-4cb7-a89d-a2e875de5b4d","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:29:01.769Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"30d18b77-c211-405d-8acb-1915676e0b21","name":"counter-6d72m-head"}
{"level":"info","ts":"2026-06-10T10:29:01.786Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"30d18b77-c211-405d-8acb-1915676e0b21","name":"counter-6d72m-workers"}
{"level":"info","ts":"2026-06-10T10:29:01.786Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"30d18b77-c211-405d-8acb-1915676e0b21","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:03.604Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d4a340b2-145f-4b05-b681-a8cd6776e9e5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:03.604Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d4a340b2-145f-4b05-b681-a8cd6776e9e5","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:03.604Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d4a340b2-145f-4b05-b681-a8cd6776e9e5","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:03.604Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d4a340b2-145f-4b05-b681-a8cd6776e9e5"}
{"level":"info","ts":"2026-06-10T10:29:03.604Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d4a340b2-145f-4b05-b681-a8cd6776e9e5","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:29:03.754Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e271551b-f199-4bec-920e-7816b8c92913"}
{"level":"info","ts":"2026-06-10T10:29:03.754Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e271551b-f199-4bec-920e-7816b8c92913"}
{"level":"info","ts":"2026-06-10T10:29:03.754Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e271551b-f199-4bec-920e-7816b8c92913","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:03.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e271551b-f199-4bec-920e-7816b8c92913","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:03.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e271551b-f199-4bec-920e-7816b8c92913","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:03.755Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e271551b-f199-4bec-920e-7816b8c92913","podName":"counter-6d72m-head-njqmv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:28:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:29:03.755Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e271551b-f199-4bec-920e-7816b8c92913","podName":"counter-6d72m-head-njqmv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:28:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:29:03.755Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e271551b-f199-4bec-920e-7816b8c92913","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:29:03.755Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"e271551b-f199-4bec-920e-7816b8c92913","seconds":300}
{"level":"info","ts":"2026-06-10T10:29:06.605Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"402db3d0-5832-4a73-a917-9aed96d1b8ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:06.605Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"402db3d0-5832-4a73-a917-9aed96d1b8ac","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:06.605Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"402db3d0-5832-4a73-a917-9aed96d1b8ac","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:06.605Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"402db3d0-5832-4a73-a917-9aed96d1b8ac"}
{"level":"info","ts":"2026-06-10T10:29:06.605Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"402db3d0-5832-4a73-a917-9aed96d1b8ac","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:29:09.605Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f6af0847-4f77-442e-be30-275dd8377468","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:09.605Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f6af0847-4f77-442e-be30-275dd8377468","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:09.606Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f6af0847-4f77-442e-be30-275dd8377468","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:09.606Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f6af0847-4f77-442e-be30-275dd8377468"}
{"level":"info","ts":"2026-06-10T10:29:09.606Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"f6af0847-4f77-442e-be30-275dd8377468","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:29:12.606Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"38aaf26f-03ba-43a9-beed-78074e4b6c57","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:12.606Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"38aaf26f-03ba-43a9-beed-78074e4b6c57","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:12.606Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"38aaf26f-03ba-43a9-beed-78074e4b6c57","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:12.606Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"38aaf26f-03ba-43a9-beed-78074e4b6c57"}
{"level":"info","ts":"2026-06-10T10:29:12.606Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"38aaf26f-03ba-43a9-beed-78074e4b6c57","RayCluster":"counter-6d72m","State":""}
{"level":"info","ts":"2026-06-10T10:29:12.770Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4bb2378b-4aa4-4479-b66c-5c0efa65c476"}
{"level":"info","ts":"2026-06-10T10:29:12.770Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4bb2378b-4aa4-4479-b66c-5c0efa65c476"}
{"level":"info","ts":"2026-06-10T10:29:12.770Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4bb2378b-4aa4-4479-b66c-5c0efa65c476","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:12.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4bb2378b-4aa4-4479-b66c-5c0efa65c476","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:12.770Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4bb2378b-4aa4-4479-b66c-5c0efa65c476","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:12.770Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4bb2378b-4aa4-4479-b66c-5c0efa65c476","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:29:12.770Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"4bb2378b-4aa4-4479-b66c-5c0efa65c476","name":"counter-6d72m","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:12Z","stateTransitionTimes":{"ready":"2026-06-10T10:29:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4abe978a-1bce-49f6-8c0a-4aa488b85043","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"463790e9-31ea-46b2-8457-4de8d21be279","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4abe978a-1bce-49f6-8c0a-4aa488b85043","pod":"counter-6d72m-head-njqmv","ip":"10.134.0.16"}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"89f85457-395f-4bd9-97b4-9080ba2ca7e1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"89f85457-395f-4bd9-97b4-9080ba2ca7e1","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4abe978a-1bce-49f6-8c0a-4aa488b85043","rayCluster":"counter-6d72m","podIPs":["10.134.0.16"]}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"89f85457-395f-4bd9-97b4-9080ba2ca7e1","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"89f85457-395f-4bd9-97b4-9080ba2ca7e1"}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"89f85457-395f-4bd9-97b4-9080ba2ca7e1","head service name":"counter-6d72m-head-svc","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"89f85457-395f-4bd9-97b4-9080ba2ca7e1","head service URL":"counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"89f85457-395f-4bd9-97b4-9080ba2ca7e1","SubmissionMode":"HTTPMode","RayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"89f85457-395f-4bd9-97b4-9080ba2ca7e1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-v546s","rayClusterName":"counter-6d72m","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:28:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-v546s","rayClusterName":"counter-6d72m","dashboardURL":"counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:28:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:29:12.799Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"89f85457-395f-4bd9-97b4-9080ba2ca7e1","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:29:12.809Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"463790e9-31ea-46b2-8457-4de8d21be279","name":"counter-6d72m-head"}
{"level":"info","ts":"2026-06-10T10:29:12.810Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d75f9b75-ee0e-461b-ba4a-e61e9347c6c6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:12.811Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d75f9b75-ee0e-461b-ba4a-e61e9347c6c6","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:12.811Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d75f9b75-ee0e-461b-ba4a-e61e9347c6c6","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:12.811Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d75f9b75-ee0e-461b-ba4a-e61e9347c6c6"}
{"level":"info","ts":"2026-06-10T10:29:12.819Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"463790e9-31ea-46b2-8457-4de8d21be279","name":"counter-6d72m-workers"}
{"level":"info","ts":"2026-06-10T10:29:12.819Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"463790e9-31ea-46b2-8457-4de8d21be279","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:12.821Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d75f9b75-ee0e-461b-ba4a-e61e9347c6c6","JobId":"counter-v546s"}
{"level":"info","ts":"2026-06-10T10:29:12.821Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"d75f9b75-ee0e-461b-ba4a-e61e9347c6c6","rayJob":"counter","jobInfo":"{\"runtime_env\":{\"env_vars\":{\"counter_name\":\"test_counter\"}},\"entrypoint_resources\":{\"R1\":2},\"entrypoint\":\"python /home/ray/jobs/counter.py\",\"submission_id\":\"counter-v546s\",\"entrypoint_num_cpus\":2,\"entrypoint_num_gpus\":2}"}
{"level":"info","ts":"2026-06-10T10:29:14.785Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"5efac34a-6157-46bb-913f-c67cae9c6f7b"}
{"level":"info","ts":"2026-06-10T10:29:14.785Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"5efac34a-6157-46bb-913f-c67cae9c6f7b"}
{"level":"info","ts":"2026-06-10T10:29:14.785Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"5efac34a-6157-46bb-913f-c67cae9c6f7b","1 head service found":"counter-6d72m-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:14.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"5efac34a-6157-46bb-913f-c67cae9c6f7b","Found 1 head Pod":"counter-6d72m-head-njqmv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:14.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"5efac34a-6157-46bb-913f-c67cae9c6f7b","head Pod":"counter-6d72m-head-njqmv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-6d72m-head-njqmv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:14.785Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"5efac34a-6157-46bb-913f-c67cae9c6f7b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:29:14.785Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"5efac34a-6157-46bb-913f-c67cae9c6f7b","seconds":300}
{"level":"info","ts":"2026-06-10T10:29:15.606Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ae4de30d-4cd4-49d7-8009-6582200af659","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:15.606Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ae4de30d-4cd4-49d7-8009-6582200af659","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:15.606Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ae4de30d-4cd4-49d7-8009-6582200af659","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:15.607Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ae4de30d-4cd4-49d7-8009-6582200af659"}
{"level":"info","ts":"2026-06-10T10:29:15.610Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ae4de30d-4cd4-49d7-8009-6582200af659","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-v546s","rayClusterName":"counter-6d72m","dashboardURL":"counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:28:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:29:13Z"},"jobId":"counter-v546s","rayClusterName":"counter-6d72m","dashboardURL":"counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:28:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:12Z","stateTransitionTimes":{"ready":"2026-06-10T10:29:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:29:15.610Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ae4de30d-4cd4-49d7-8009-6582200af659","old JobStatus":"","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:29:15.623Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ead13038-ac65-4bd7-8c19-951e617c607d","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:15.623Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ead13038-ac65-4bd7-8c19-951e617c607d","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:15.623Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ead13038-ac65-4bd7-8c19-951e617c607d","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:15.623Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ead13038-ac65-4bd7-8c19-951e617c607d"}
{"level":"info","ts":"2026-06-10T10:29:15.626Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"ead13038-ac65-4bd7-8c19-951e617c607d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:29:13Z"},"jobId":"counter-v546s","rayClusterName":"counter-6d72m","dashboardURL":"counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:28:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:12Z","stateTransitionTimes":{"ready":"2026-06-10T10:29:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:29:13Z"},"jobId":"counter-v546s","rayClusterName":"counter-6d72m","dashboardURL":"counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:28:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:12Z","stateTransitionTimes":{"ready":"2026-06-10T10:29:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:29:18.624Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"dddb67b7-0dd2-45e6-8827-7913fe1f4c37","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:18.624Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"dddb67b7-0dd2-45e6-8827-7913fe1f4c37","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.624Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"dddb67b7-0dd2-45e6-8827-7913fe1f4c37","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:18.624Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"dddb67b7-0dd2-45e6-8827-7913fe1f4c37"}
{"level":"info","ts":"2026-06-10T10:29:18.627Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"dddb67b7-0dd2-45e6-8827-7913fe1f4c37","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:29:13Z"},"jobId":"counter-v546s","rayClusterName":"counter-6d72m","dashboardURL":"counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:28:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:12Z","stateTransitionTimes":{"ready":"2026-06-10T10:29:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:29:13Z","endTime":"2026-06-10T10:29:18Z"},"jobId":"counter-v546s","rayClusterName":"counter-6d72m","dashboardURL":"counter-6d72m-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-10T10:28:51Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:12Z","stateTransitionTimes":{"ready":"2026-06-10T10:29:12Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.16","serviceIP":"10.134.0.16","podName":"counter-6d72m-head-njqmv","serviceName":"counter-6d72m-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:29:12Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:29:18.627Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"dddb67b7-0dd2-45e6-8827-7913fe1f4c37","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-10T10:29:18.641Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"e4791183-d583-49e8-b30e-6a654de6523e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:18.641Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"e4791183-d583-49e8-b30e-6a654de6523e","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:29:18 +0000 UTC","Now":"2026-06-10T10:29:18.641Z","ShutdownTime":"2026-06-10T10:29:18.000Z"}
{"level":"info","ts":"2026-06-10T10:29:18.641Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"e4791183-d583-49e8-b30e-6a654de6523e","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:29:18.649Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"e4791183-d583-49e8-b30e-6a654de6523e","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:18.649Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"e4791183-d583-49e8-b30e-6a654de6523e","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:29:18.649Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"e4791183-d583-49e8-b30e-6a654de6523e","RayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"a51840e3-58de-4a01-aac6-1b0e95d3cc1c"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"b1ef10bf-64e3-4b2a-92d7-49601b83a99b","rayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4ad49cab-aa31-4051-930f-aa219974f214","namespacedName":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4ad49cab-aa31-4051-930f-aa219974f214","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4ad49cab-aa31-4051-930f-aa219974f214","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"reconcileID":"417fce31-f388-488d-b55d-a6a5fff023de"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"b1ef10bf-64e3-4b2a-92d7-49601b83a99b","pod":"counter-6d72m-head-njqmv","ip":"10.134.0.16"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4ad49cab-aa31-4051-930f-aa219974f214","namespace":"test-ns-bbzgk","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"b1ef10bf-64e3-4b2a-92d7-49601b83a99b","rayCluster":"counter-6d72m","podIPs":["10.134.0.16"]}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"57888a64-9b0e-4041-ab1e-1678d520d958","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"57888a64-9b0e-4041-ab1e-1678d520d958","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:29:18 +0000 UTC","Now":"2026-06-10T10:29:18.650Z","ShutdownTime":"2026-06-10T10:29:18.000Z"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"57888a64-9b0e-4041-ab1e-1678d520d958","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"57888a64-9b0e-4041-ab1e-1678d520d958","RayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"57888a64-9b0e-4041-ab1e-1678d520d958","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:29:18.650Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"57888a64-9b0e-4041-ab1e-1678d520d958","RayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.657Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4ad49cab-aa31-4051-930f-aa219974f214","name":"kuberay-gateway-access","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:29:18.662Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4ad49cab-aa31-4051-930f-aa219974f214","name":"test-ns-bbzgk-counter-6d72m","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:29:18.667Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4ad49cab-aa31-4051-930f-aa219974f214","configMap":"kube-rbac-proxy-config-counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.671Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4ad49cab-aa31-4051-930f-aa219974f214","serviceAccount":"counter-6d72m-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:29:18.679Z","logger":"raycluster-resource","msg":"default","name":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.679Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-6d72m","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:29:18.690Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"e0b8c92c-c269-4e15-89aa-b46278c95578","namespace":"test-ns-bbzgk","clusterName":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.690Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"4ad49cab-aa31-4051-930f-aa219974f214","cluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.690Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"51c57832-00cd-41a3-a5e1-87a30c2ab31f","namespacedName":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:18.690Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"51c57832-00cd-41a3-a5e1-87a30c2ab31f"}
{"level":"info","ts":"2026-06-10T10:29:18.690Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"51c57832-00cd-41a3-a5e1-87a30c2ab31f","namespace":"test-ns-bbzgk","authClusters":0}
{"level":"info","ts":"2026-06-10T10:29:18.690Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"51c57832-00cd-41a3-a5e1-87a30c2ab31f","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:29:18.690Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"783ad454-0a15-4bf7-a485-e90f5afe3c32"}
{"level":"info","ts":"2026-06-10T10:29:18.691Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6397e0f0-14cc-490e-81d3-e20013212d8c","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:18.691Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6397e0f0-14cc-490e-81d3-e20013212d8c","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:29:18 +0000 UTC","Now":"2026-06-10T10:29:18.691Z","ShutdownTime":"2026-06-10T10:29:18.000Z"}
{"level":"info","ts":"2026-06-10T10:29:18.691Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6397e0f0-14cc-490e-81d3-e20013212d8c","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:29:18.699Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6397e0f0-14cc-490e-81d3-e20013212d8c","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:18.699Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6397e0f0-14cc-490e-81d3-e20013212d8c","isClusterDeleted":true}
{"level":"info","ts":"2026-06-10T10:29:18.699Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"6397e0f0-14cc-490e-81d3-e20013212d8c","RayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.699Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"1a90fa87-24c5-443c-9e9b-1d9a932fa9b0","namespacedName":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:18.699Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"1a90fa87-24c5-443c-9e9b-1d9a932fa9b0"}
{"level":"info","ts":"2026-06-10T10:29:18.699Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"1a90fa87-24c5-443c-9e9b-1d9a932fa9b0","namespace":"test-ns-bbzgk","authClusters":0}
{"level":"info","ts":"2026-06-10T10:29:18.699Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"1a90fa87-24c5-443c-9e9b-1d9a932fa9b0","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:29:18.699Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"df356a07-e34b-4a53-b791-ca2f9b1a8227"}
{"level":"info","ts":"2026-06-10T10:29:18.708Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"d422f388-964a-446c-bd8e-d7f9e9411f71"}
{"level":"info","ts":"2026-06-10T10:29:18.710Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"463f8f52-8a7f-414c-b949-aeac09dc764d","namespacedName":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:18.710Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"463f8f52-8a7f-414c-b949-aeac09dc764d"}
{"level":"info","ts":"2026-06-10T10:29:18.710Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"463f8f52-8a7f-414c-b949-aeac09dc764d","namespace":"test-ns-bbzgk","authClusters":0}
{"level":"info","ts":"2026-06-10T10:29:18.710Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"463f8f52-8a7f-414c-b949-aeac09dc764d","namespace":"test-ns-bbzgk"}
{"level":"error","ts":"2026-06-10T10:29:18.711Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"e0b8c92c-c269-4e15-89aa-b46278c95578","certificate":"ray-head-cert-counter-6d72m","error":"certificates.cert-manager.io \"ray-head-cert-counter-6d72m\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:18.728Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"e0b8c92c-c269-4e15-89aa-b46278c95578","errorCount":1}
{"level":"info","ts":"2026-06-10T10:29:18.728Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"e0b8c92c-c269-4e15-89aa-b46278c95578"}
{"level":"error","ts":"2026-06-10T10:29:18.728Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"e0b8c92c-c269-4e15-89aa-b46278c95578","error":"certificates.cert-manager.io \"ray-head-cert-counter-6d72m\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:18.734Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"8ce7ebc0-5d71-4b8e-a192-b468450b9d45","namespace":"test-ns-bbzgk","clusterName":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:18.734Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"counter-6d72m","reconcileID":"8ce7ebc0-5d71-4b8e-a192-b468450b9d45","namespace":"test-ns-bbzgk","clusterName":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:19.549Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"426fced9-be0a-4bb2-86fe-e74b822f7d5d","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:19.549Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"426fced9-be0a-4bb2-86fe-e74b822f7d5d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:29:19.558Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"426fced9-be0a-4bb2-86fe-e74b822f7d5d"}
{"level":"info","ts":"2026-06-10T10:29:19.558Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"426fced9-be0a-4bb2-86fe-e74b822f7d5d","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:29:19.558Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"426fced9-be0a-4bb2-86fe-e74b822f7d5d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-68dpl","rayClusterName":"fail-hxn94","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:29:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:29:19.558Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"426fced9-be0a-4bb2-86fe-e74b822f7d5d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:29:19.569Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"388a5e3b-c1b4-4801-aaf3-94620d6fae2a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:19.569Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"388a5e3b-c1b4-4801-aaf3-94620d6fae2a","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.569Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"388a5e3b-c1b4-4801-aaf3-94620d6fae2a","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.569Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"388a5e3b-c1b4-4801-aaf3-94620d6fae2a","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.573Z","logger":"raycluster-resource","msg":"default","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.573Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-hxn94","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"388a5e3b-c1b4-4801-aaf3-94620d6fae2a","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"5c1c63ac-5dd4-4498-bef7-29badb536d54","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","namespacedName":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","mode":"IntegratedOAuth","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"388a5e3b-c1b4-4801-aaf3-94620d6fae2a"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"388a5e3b-c1b4-4801-aaf3-94620d6fae2a","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"af2694de-11ce-4814-b4ee-2a35ef6d4c76","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:19.589Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"af2694de-11ce-4814-b4ee-2a35ef6d4c76","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"af2694de-11ce-4814-b4ee-2a35ef6d4c76","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.590Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"af2694de-11ce-4814-b4ee-2a35ef6d4c76"}
{"level":"info","ts":"2026-06-10T10:29:19.590Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"af2694de-11ce-4814-b4ee-2a35ef6d4c76","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:19.594Z","logger":"raycluster-resource","msg":"default","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.594Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-hxn94","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:29:19.596Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","name":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:19.597Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","error":"head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"error","ts":"2026-06-10T10:29:19.597Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.597Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"info","ts":"2026-06-10T10:29:19.597Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.597Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.597Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.597Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:29:19.597Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","name":"fail-hxn94","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:29:19.603Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85d64480-7051-43aa-990e-dc864d675b8a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:19.603Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85d64480-7051-43aa-990e-dc864d675b8a","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.603Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85d64480-7051-43aa-990e-dc864d675b8a","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.603Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85d64480-7051-43aa-990e-dc864d675b8a"}
{"level":"info","ts":"2026-06-10T10:29:19.603Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85d64480-7051-43aa-990e-dc864d675b8a","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:19.605Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"5c1c63ac-5dd4-4498-bef7-29badb536d54","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.606Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"991547b8-53f0-4b5d-90e7-41b25d06797d","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.606Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.606Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","name":"fail-hxn94","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-hxn94\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"fail-hxn94"}}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764"}
{"level":"error","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"35027046-9585-457a-9b54-644ef6415764","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07"}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07"}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","error":"head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"error","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:29:19.610Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","name":"fail-hxn94","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:29:19.611Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","name":"fail-hxn94-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:29:19.615Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b"}
{"level":"info","ts":"2026-06-10T10:29:19.615Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b","rayCluster":"fail-hxn94","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:29:19.615Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.619Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","name":"kuberay-gateway-access","namespace":"test-ns-bbzgk","operation":"created"}
{"level":"info","ts":"2026-06-10T10:29:19.621Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"991547b8-53f0-4b5d-90e7-41b25d06797d","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:29:19.622Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07"}
{"level":"error","ts":"2026-06-10T10:29:19.622Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"2d75ffb3-2493-4fa9-afc6-a41cd2acbd07","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.622Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306"}
{"level":"info","ts":"2026-06-10T10:29:19.622Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306"}
{"level":"info","ts":"2026-06-10T10:29:19.622Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:19.622Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","error":"head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"error","ts":"2026-06-10T10:29:19.622Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.622Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6fc33e10-5833-4d43-908e-98d23e56e086","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6fc33e10-5833-4d43-908e-98d23e56e086","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6fc33e10-5833-4d43-908e-98d23e56e086","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6fc33e10-5833-4d43-908e-98d23e56e086"}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6fc33e10-5833-4d43-908e-98d23e56e086","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:19.623Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","name":"fail-hxn94","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:29:19.699Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.699Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","name":"test-ns-bbzgk-fail-hxn94","operation":"created"}
{"level":"info","ts":"2026-06-10T10:29:19.703Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"991547b8-53f0-4b5d-90e7-41b25d06797d","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:29:19.703Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"991547b8-53f0-4b5d-90e7-41b25d06797d","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.703Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"41eb9d68-5d51-445f-81fa-02ceccdcc92e","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.706Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","name":"kube-rbac-proxy-config-fail-hxn94","operation":"created"}
{"level":"info","ts":"2026-06-10T10:29:19.706Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","name":"fail-hxn94","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-hxn94\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"fail-hxn94"}}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306"}
{"level":"error","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7f341cc5-1495-4c94-bd96-986da73fd306","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c"}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c"}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c","error":"head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"error","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c"}
{"level":"error","ts":"2026-06-10T10:29:19.707Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1f6698d4-8315-41ff-a817-98906aea406c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.710Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b"}
{"level":"info","ts":"2026-06-10T10:29:19.710Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b","rayCluster":"fail-hxn94","podIPs":[]}
{"level":"error","ts":"2026-06-10T10:29:19.710Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-hxn94\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.710Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c"}
{"level":"error","ts":"2026-06-10T10:29:19.710Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b29fbd8a-d1b0-4565-a8cc-31c1cd872a3c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-hxn94\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.711Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd57be85-1290-41c8-ad09-62ea99c53e08","namespacedName":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.711Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd57be85-1290-41c8-ad09-62ea99c53e08","mode":"IntegratedOAuth","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.711Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd57be85-1290-41c8-ad09-62ea99c53e08","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.713Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"41eb9d68-5d51-445f-81fa-02ceccdcc92e","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:29:19.717Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b"}
{"level":"info","ts":"2026-06-10T10:29:19.718Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b","rayCluster":"fail-hxn94","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:29:19.718Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f209e992-b7c7-44ae-a0a4-7aa62e9d674b"}
{"level":"info","ts":"2026-06-10T10:29:19.718Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dcc6d253-8fce-4e93-8a18-fa804fc540fa","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.718Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dcc6d253-8fce-4e93-8a18-fa804fc540fa"}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"41eb9d68-5d51-445f-81fa-02ceccdcc92e","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"41eb9d68-5d51-445f-81fa-02ceccdcc92e","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7"}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7"}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7","error":"head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd57be85-1290-41c8-ad09-62ea99c53e08","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b21a4487-278d-439f-b793-da7ac1ab8183","namespacedName":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b21a4487-278d-439f-b793-da7ac1ab8183","mode":"IntegratedOAuth","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.799Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b21a4487-278d-439f-b793-da7ac1ab8183","cluster":"fail-hxn94"}
{"level":"error","ts":"2026-06-10T10:29:19.799Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7"}
{"level":"error","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"86ef5f84-3391-4175-a1d0-18f2954c97a7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f9238704-5c6a-4ea5-85ff-a6b32c141a03","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f9238704-5c6a-4ea5-85ff-a6b32c141a03","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9cc086bf-6962-4f45-b8e7-16a09357533a","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f9238704-5c6a-4ea5-85ff-a6b32c141a03","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f11148d2-0d8e-46f4-9fca-b7b9408eff2d","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9cc086bf-6962-4f45-b8e7-16a09357533a"}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f9238704-5c6a-4ea5-85ff-a6b32c141a03"}
{"level":"info","ts":"2026-06-10T10:29:19.800Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f9238704-5c6a-4ea5-85ff-a6b32c141a03","RayCluster":"fail-hxn94","State":""}
{"level":"error","ts":"2026-06-10T10:29:19.807Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b21a4487-278d-439f-b793-da7ac1ab8183","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-hxn94\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.807Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b21a4487-278d-439f-b793-da7ac1ab8183"}
{"level":"error","ts":"2026-06-10T10:29:19.807Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"b21a4487-278d-439f-b793-da7ac1ab8183","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-hxn94\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.812Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"8cc7d635-0341-43cf-8677-ed8fbcdb8f24","namespacedName":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:19.812Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"8cc7d635-0341-43cf-8677-ed8fbcdb8f24","mode":"IntegratedOAuth","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.812Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"8cc7d635-0341-43cf-8677-ed8fbcdb8f24","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.812Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f11148d2-0d8e-46f4-9fca-b7b9408eff2d","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:29:19.820Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f11148d2-0d8e-46f4-9fca-b7b9408eff2d","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:29:19.820Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f11148d2-0d8e-46f4-9fca-b7b9408eff2d","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6"}
{"level":"info","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6"}
{"level":"info","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6","error":"head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"error","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"info","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6"}
{"level":"error","ts":"2026-06-10T10:29:19.881Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c989e0ec-1808-46b2-85b4-c5773dcd0dc6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:19.899Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"8cc7d635-0341-43cf-8677-ed8fbcdb8f24","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:20.041Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37"}
{"level":"info","ts":"2026-06-10T10:29:20.041Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37"}
{"level":"info","ts":"2026-06-10T10:29:20.041Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:20.041Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37","error":"head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"error","ts":"2026-06-10T10:29:20.041Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:20.041Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"info","ts":"2026-06-10T10:29:20.042Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:20.042Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:20.042Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:20.042Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37"}
{"level":"error","ts":"2026-06-10T10:29:20.042Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b822e3f3-919a-4984-914f-20788eb33a37","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:20.363Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f"}
{"level":"info","ts":"2026-06-10T10:29:20.363Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f"}
{"level":"info","ts":"2026-06-10T10:29:20.363Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:20.364Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f","error":"head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"error","ts":"2026-06-10T10:29:20.364Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:20.364Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found"}
{"level":"info","ts":"2026-06-10T10:29:20.364Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:20.364Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:20.364Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:29:20.364Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f"}
{"level":"error","ts":"2026-06-10T10:29:20.364Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"881683ac-8564-41fb-9386-99c36f3afd8f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-hxn94 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e"}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e"}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e"}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e"}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","mode":"IntegratedOAuth","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","cluster":"fail-hxn94","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","cluster":"fail-hxn94","authType":"IntegratedOAuth","serviceAccount":"fail-hxn94-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-hxn94","ray.io/group":"headgroup","ray.io/identifier":"fail-hxn94-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:29:21.005Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","enabled":true}
{"level":"info","ts":"2026-06-10T10:29:21.022Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","name":"fail-hxn94-head-nj5fr"}
{"level":"info","ts":"2026-06-10T10:29:21.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:29:21.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:29:21.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:29:21.022Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e"}
{"level":"info","ts":"2026-06-10T10:29:21.023Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","nodeType":"worker","rayStartParams":{"address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:29:21.023Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","rayStartCmd":"ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:29:21.023Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:29:21.023Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","enabled":true}
{"level":"info","ts":"2026-06-10T10:29:21.038Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","name":"fail-hxn94-small-group-worker-s48tx"}
{"level":"info","ts":"2026-06-10T10:29:21.038Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.038Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-hxn94-head-svc"},"newHead":{"podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"}}
{"level":"info","ts":"2026-06-10T10:29:21.038Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"21f6c1d5-2c67-489b-9683-cbad1b8b7a6e","name":"fail-hxn94","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8"}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8"}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"2a6cdb1e-8db8-4959-b3ca-b9099d5c2d9f","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"6a9fd8d4-ed7a-4608-97f5-ec767e3b8ea5","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"6a9fd8d4-ed7a-4608-97f5-ec767e3b8ea5"}
{"level":"info","ts":"2026-06-10T10:29:21.110Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ccf75f67-b594-4ca6-89cf-8a1c49df6374","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:21.111Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ccf75f67-b594-4ca6-89cf-8a1c49df6374","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.111Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ccf75f67-b594-4ca6-89cf-8a1c49df6374","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:21.111Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.111Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.111Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ccf75f67-b594-4ca6-89cf-8a1c49df6374"}
{"level":"info","ts":"2026-06-10T10:29:21.111Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ccf75f67-b594-4ca6-89cf-8a1c49df6374","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:21.111Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-hxn94-head-svc"},"newHead":{"podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"}}
{"level":"info","ts":"2026-06-10T10:29:21.111Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","name":"fail-hxn94","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:29:21.118Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","name":"fail-hxn94","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-hxn94\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"fail-hxn94"}}
{"level":"info","ts":"2026-06-10T10:29:21.118Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8"}
{"level":"error","ts":"2026-06-10T10:29:21.118Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b5235cc0-6656-4ca0-a741-f733b468a6b8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-hxn94\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:29:21.123Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71"}
{"level":"info","ts":"2026-06-10T10:29:21.123Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71"}
{"level":"info","ts":"2026-06-10T10:29:21.123Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:21.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:21.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:29:21.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:29:21.124Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.124Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.124Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"2a6cdb1e-8db8-4959-b3ca-b9099d5c2d9f","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:29:21.124Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.124Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:29:21.124Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1e886e41-68df-41f5-b432-9b07c0beac71","name":"fail-hxn94","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:29:21.134Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"2a6cdb1e-8db8-4959-b3ca-b9099d5c2d9f","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:29:21.134Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"2a6cdb1e-8db8-4959-b3ca-b9099d5c2d9f","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.136Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"602e0582-62d6-40ef-abd5-51238f247e30","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.136Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bdc9a78c-a776-4ae0-96a4-419b1a3ff150","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:21.136Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bdc9a78c-a776-4ae0-96a4-419b1a3ff150","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.136Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bdc9a78c-a776-4ae0-96a4-419b1a3ff150","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:21.136Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bdc9a78c-a776-4ae0-96a4-419b1a3ff150"}
{"level":"info","ts":"2026-06-10T10:29:21.136Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bdc9a78c-a776-4ae0-96a4-419b1a3ff150","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:21.144Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"602e0582-62d6-40ef-abd5-51238f247e30","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:29:21.155Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"602e0582-62d6-40ef-abd5-51238f247e30","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:29:21.155Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"602e0582-62d6-40ef-abd5-51238f247e30","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.199Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"43e6bd28-17d6-42c2-9d95-79cbdc6472de","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.199Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"43e6bd28-17d6-42c2-9d95-79cbdc6472de"}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8"}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8"}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.468Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.469Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:29:21.469Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"e4074e1a-7de1-4abc-8796-e32c9bd653f8","seconds":300}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2"}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2"}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.500Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.501Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:29:21.501Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b9277fb7-210c-44f4-bc18-789da85a0dd2","seconds":300}
{"level":"info","ts":"2026-06-10T10:29:21.640Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"3f020bab-3b6b-41d4-a113-99a9d1456fbf","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:21.640Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"3f020bab-3b6b-41d4-a113-99a9d1456fbf","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:29:18 +0000 UTC","Now":"2026-06-10T10:29:21.640Z","ShutdownTime":"2026-06-10T10:29:18.000Z"}
{"level":"info","ts":"2026-06-10T10:29:21.640Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"3f020bab-3b6b-41d4-a113-99a9d1456fbf","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:29:21.640Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"3f020bab-3b6b-41d4-a113-99a9d1456fbf","RayCluster":{"name":"counter-6d72m","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:21.640Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"3f020bab-3b6b-41d4-a113-99a9d1456fbf","isClusterDeleted":true}
{"level":"info","ts":"2026-06-10T10:29:21.640Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"3f020bab-3b6b-41d4-a113-99a9d1456fbf","RayCluster":"counter-6d72m"}
{"level":"info","ts":"2026-06-10T10:29:21.815Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b"}
{"level":"info","ts":"2026-06-10T10:29:21.815Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b"}
{"level":"info","ts":"2026-06-10T10:29:21.815Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:21.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:21.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:29:21.816Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:21.816Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.816Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:29:21.816Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:21.816Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.816Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:21.816Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"newHead":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"}}
{"level":"info","ts":"2026-06-10T10:29:21.816Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"98a364b6-2e64-4b9c-bb57-146db3a65e3b","name":"fail-hxn94","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:29:21.829Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"be448d9e-eda1-4b0a-9035-1881f5460cfa","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.829Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"ea23755d-c897-4ca1-ba36-f0c2e8123398","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.829Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"be448d9e-eda1-4b0a-9035-1881f5460cfa","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:29:21.829Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30cb7f96-eb88-4c1a-bff9-64648700cd8c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:21.829Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30cb7f96-eb88-4c1a-bff9-64648700cd8c","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.829Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30cb7f96-eb88-4c1a-bff9-64648700cd8c","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:21.830Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30cb7f96-eb88-4c1a-bff9-64648700cd8c"}
{"level":"info","ts":"2026-06-10T10:29:21.830Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30cb7f96-eb88-4c1a-bff9-64648700cd8c","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:21.840Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"be448d9e-eda1-4b0a-9035-1881f5460cfa","certificate":"ray-head-cert-fail-hxn94","dnsNames":["fail-hxn94-head-svc","localhost","fail-hxn94-head-svc.test-ns-bbzgk.svc","fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:29:21.841Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"ea23755d-c897-4ca1-ba36-f0c2e8123398","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:29:21.852Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"ea23755d-c897-4ca1-ba36-f0c2e8123398","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:29:21.852Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"ea23755d-c897-4ca1-ba36-f0c2e8123398","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:21.852Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"be448d9e-eda1-4b0a-9035-1881f5460cfa","certificate":"ray-worker-cert-fail-hxn94","dnsNames":["fail-hxn94-worker-svc","localhost","fail-hxn94-small-group","fail-hxn94-worker-svc.test-ns-bbzgk.svc","fail-hxn94-worker-svc.test-ns-bbzgk.svc.cluster.local","*.fail-hxn94-worker-svc.test-ns-bbzgk.svc","*.fail-hxn94-worker-svc.test-ns-bbzgk.svc.cluster.local","*-worker-*.test-ns-bbzgk.svc","*-worker-*.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:29:21.853Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"be448d9e-eda1-4b0a-9035-1881f5460cfa","rayCluster":"fail-hxn94","podIPs":["10.134.0.17"]}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395"}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395"}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:29:22.455Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"d18ff365-9311-4e98-991d-143f8f520395","seconds":300}
{"level":"info","ts":"2026-06-10T10:29:22.570Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"52beccbd-859e-4e92-ae79-cb7006885154","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:22.570Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"52beccbd-859e-4e92-ae79-cb7006885154","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:22.570Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"52beccbd-859e-4e92-ae79-cb7006885154","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:22.570Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"52beccbd-859e-4e92-ae79-cb7006885154"}
{"level":"info","ts":"2026-06-10T10:29:22.570Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"52beccbd-859e-4e92-ae79-cb7006885154","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4"}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4"}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","podName":"fail-hxn94-head-nj5fr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:29:23.136Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"498dc14d-d57e-4e00-99ed-273d1330a0f4","seconds":300}
{"level":"info","ts":"2026-06-10T10:29:25.570Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ee176f71-61f6-4787-83f3-25c59bf33503","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:25.570Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ee176f71-61f6-4787-83f3-25c59bf33503","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:25.570Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ee176f71-61f6-4787-83f3-25c59bf33503","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:25.571Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ee176f71-61f6-4787-83f3-25c59bf33503"}
{"level":"info","ts":"2026-06-10T10:29:25.571Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ee176f71-61f6-4787-83f3-25c59bf33503","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:28.571Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"32082a0f-123e-4af6-b969-23452f10b7ec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:28.571Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"32082a0f-123e-4af6-b969-23452f10b7ec","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:28.571Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"32082a0f-123e-4af6-b969-23452f10b7ec","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:28.571Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"32082a0f-123e-4af6-b969-23452f10b7ec"}
{"level":"info","ts":"2026-06-10T10:29:28.571Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"32082a0f-123e-4af6-b969-23452f10b7ec","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:31.571Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ecb3a034-9163-4e3a-8333-e42a85dc64b9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:31.572Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ecb3a034-9163-4e3a-8333-e42a85dc64b9","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:31.572Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ecb3a034-9163-4e3a-8333-e42a85dc64b9","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:31.572Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ecb3a034-9163-4e3a-8333-e42a85dc64b9"}
{"level":"info","ts":"2026-06-10T10:29:31.572Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ecb3a034-9163-4e3a-8333-e42a85dc64b9","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:34.573Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"170b7fce-4dc1-4b71-b32d-00e122b2680f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:34.573Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"170b7fce-4dc1-4b71-b32d-00e122b2680f","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:34.573Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"170b7fce-4dc1-4b71-b32d-00e122b2680f","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:34.573Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"170b7fce-4dc1-4b71-b32d-00e122b2680f"}
{"level":"info","ts":"2026-06-10T10:29:34.573Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"170b7fce-4dc1-4b71-b32d-00e122b2680f","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:37.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d001348e-d651-445d-8d72-7b5093d802d1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:37.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d001348e-d651-445d-8d72-7b5093d802d1","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:37.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d001348e-d651-445d-8d72-7b5093d802d1","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:37.574Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d001348e-d651-445d-8d72-7b5093d802d1"}
{"level":"info","ts":"2026-06-10T10:29:37.574Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d001348e-d651-445d-8d72-7b5093d802d1","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:40.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"89d0547d-e54d-4a1f-8780-499d4578091b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:40.575Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"89d0547d-e54d-4a1f-8780-499d4578091b","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:40.575Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"89d0547d-e54d-4a1f-8780-499d4578091b","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:40.575Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"89d0547d-e54d-4a1f-8780-499d4578091b"}
{"level":"info","ts":"2026-06-10T10:29:40.575Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"89d0547d-e54d-4a1f-8780-499d4578091b","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:43.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d612782-c6c1-4951-a501-85cbf6940adb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:43.575Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d612782-c6c1-4951-a501-85cbf6940adb","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:43.575Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d612782-c6c1-4951-a501-85cbf6940adb","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:43.576Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d612782-c6c1-4951-a501-85cbf6940adb"}
{"level":"info","ts":"2026-06-10T10:29:43.576Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d612782-c6c1-4951-a501-85cbf6940adb","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:46.577Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d0087899-85af-49d6-a7f4-7cd0c11e3417","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:46.577Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d0087899-85af-49d6-a7f4-7cd0c11e3417","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:46.577Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d0087899-85af-49d6-a7f4-7cd0c11e3417","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:46.577Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d0087899-85af-49d6-a7f4-7cd0c11e3417"}
{"level":"info","ts":"2026-06-10T10:29:46.577Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d0087899-85af-49d6-a7f4-7cd0c11e3417","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:46.905Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f2737ce3-c8ee-4304-b881-9d9541ab2282","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:29:46.906Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f2737ce3-c8ee-4304-b881-9d9541ab2282","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:29:46.906Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f2737ce3-c8ee-4304-b881-9d9541ab2282","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:29:46.906Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"f2737ce3-c8ee-4304-b881-9d9541ab2282","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:29:49.578Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"b8f64cb2-650c-40fa-bb9f-6d53abc370bf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:49.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"b8f64cb2-650c-40fa-bb9f-6d53abc370bf","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:49.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"b8f64cb2-650c-40fa-bb9f-6d53abc370bf","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:49.578Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"b8f64cb2-650c-40fa-bb9f-6d53abc370bf"}
{"level":"info","ts":"2026-06-10T10:29:49.578Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"b8f64cb2-650c-40fa-bb9f-6d53abc370bf","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:49.718Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cbc73464-4d68-4d49-a43e-db16f9d6c292","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:49.718Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cbc73464-4d68-4d49-a43e-db16f9d6c292","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:29:49.718Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cbc73464-4d68-4d49-a43e-db16f9d6c292","pod":"fail-hxn94-small-group-worker-s48tx","ip":"10.132.0.42"}
{"level":"info","ts":"2026-06-10T10:29:49.730Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cbc73464-4d68-4d49-a43e-db16f9d6c292","certificate":"ray-head-cert-fail-hxn94","dnsNames":["fail-hxn94-head-svc","localhost","fail-hxn94-head-svc.test-ns-bbzgk.svc","fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.17","10.132.0.42","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:29:49.742Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cbc73464-4d68-4d49-a43e-db16f9d6c292","certificate":"ray-worker-cert-fail-hxn94","dnsNames":["fail-hxn94-worker-svc","localhost","fail-hxn94-small-group","fail-hxn94-worker-svc.test-ns-bbzgk.svc","fail-hxn94-worker-svc.test-ns-bbzgk.svc.cluster.local","*.fail-hxn94-worker-svc.test-ns-bbzgk.svc","*.fail-hxn94-worker-svc.test-ns-bbzgk.svc.cluster.local","*-worker-*.test-ns-bbzgk.svc","*-worker-*.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.17","10.132.0.42","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:29:49.742Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cbc73464-4d68-4d49-a43e-db16f9d6c292","rayCluster":"fail-hxn94","podIPs":["10.134.0.17","10.132.0.42"]}
{"level":"info","ts":"2026-06-10T10:29:52.578Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"df6ac4e0-0db6-44b3-b940-289d58c8da96","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:52.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"df6ac4e0-0db6-44b3-b940-289d58c8da96","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:52.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"df6ac4e0-0db6-44b3-b940-289d58c8da96","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:52.578Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"df6ac4e0-0db6-44b3-b940-289d58c8da96"}
{"level":"info","ts":"2026-06-10T10:29:52.578Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"df6ac4e0-0db6-44b3-b940-289d58c8da96","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:55.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d4411fd7-dd72-480b-9531-c4c57e151c93","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:55.579Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d4411fd7-dd72-480b-9531-c4c57e151c93","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:55.579Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d4411fd7-dd72-480b-9531-c4c57e151c93","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:55.579Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d4411fd7-dd72-480b-9531-c4c57e151c93"}
{"level":"info","ts":"2026-06-10T10:29:55.579Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d4411fd7-dd72-480b-9531-c4c57e151c93","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:29:58.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"114fa8c2-e2c6-4a38-907d-886ef0064ed1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:29:58.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"114fa8c2-e2c6-4a38-907d-886ef0064ed1","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:29:58.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"114fa8c2-e2c6-4a38-907d-886ef0064ed1","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:29:58.580Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"114fa8c2-e2c6-4a38-907d-886ef0064ed1"}
{"level":"info","ts":"2026-06-10T10:29:58.580Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"114fa8c2-e2c6-4a38-907d-886ef0064ed1","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:01.581Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"48bb228a-5d77-4e9e-bcba-fa6a9b3ad417","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:01.581Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"48bb228a-5d77-4e9e-bcba-fa6a9b3ad417","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:01.581Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"48bb228a-5d77-4e9e-bcba-fa6a9b3ad417","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:01.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"48bb228a-5d77-4e9e-bcba-fa6a9b3ad417"}
{"level":"info","ts":"2026-06-10T10:30:01.581Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"48bb228a-5d77-4e9e-bcba-fa6a9b3ad417","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:04.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"09bc4e9a-e7e2-4208-bfde-9b01a40ecff4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:04.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"09bc4e9a-e7e2-4208-bfde-9b01a40ecff4","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:04.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"09bc4e9a-e7e2-4208-bfde-9b01a40ecff4","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:04.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"09bc4e9a-e7e2-4208-bfde-9b01a40ecff4"}
{"level":"info","ts":"2026-06-10T10:30:04.582Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"09bc4e9a-e7e2-4208-bfde-9b01a40ecff4","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:07.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"04cdd072-8b16-425c-9a91-28143d4cef80","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:07.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"04cdd072-8b16-425c-9a91-28143d4cef80","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:07.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"04cdd072-8b16-425c-9a91-28143d4cef80","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:07.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"04cdd072-8b16-425c-9a91-28143d4cef80"}
{"level":"info","ts":"2026-06-10T10:30:07.583Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"04cdd072-8b16-425c-9a91-28143d4cef80","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:10.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6d3744b3-0d48-4bbd-94b3-7e4946cfe672","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:10.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6d3744b3-0d48-4bbd-94b3-7e4946cfe672","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:10.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6d3744b3-0d48-4bbd-94b3-7e4946cfe672","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:10.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6d3744b3-0d48-4bbd-94b3-7e4946cfe672"}
{"level":"info","ts":"2026-06-10T10:30:10.583Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6d3744b3-0d48-4bbd-94b3-7e4946cfe672","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:13.584Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"415ab425-fbf3-4e69-bd01-b1b0f5b86db7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:13.584Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"415ab425-fbf3-4e69-bd01-b1b0f5b86db7","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:13.584Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"415ab425-fbf3-4e69-bd01-b1b0f5b86db7","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:13.584Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"415ab425-fbf3-4e69-bd01-b1b0f5b86db7"}
{"level":"info","ts":"2026-06-10T10:30:13.584Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"415ab425-fbf3-4e69-bd01-b1b0f5b86db7","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:16.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"38a24b4c-8083-4628-a2cd-f6f6772d63ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:16.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"38a24b4c-8083-4628-a2cd-f6f6772d63ff","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:16.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"38a24b4c-8083-4628-a2cd-f6f6772d63ff","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:16.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"38a24b4c-8083-4628-a2cd-f6f6772d63ff"}
{"level":"info","ts":"2026-06-10T10:30:16.586Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"38a24b4c-8083-4628-a2cd-f6f6772d63ff","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:19.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"a36c4645-c8eb-48c0-8c80-d7da52004103","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:19.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"a36c4645-c8eb-48c0-8c80-d7da52004103","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:19.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"a36c4645-c8eb-48c0-8c80-d7da52004103","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:19.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"a36c4645-c8eb-48c0-8c80-d7da52004103"}
{"level":"info","ts":"2026-06-10T10:30:19.586Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"a36c4645-c8eb-48c0-8c80-d7da52004103","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:22.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ef0447bc-9a75-45ad-8dca-b64d1252cb95","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:22.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ef0447bc-9a75-45ad-8dca-b64d1252cb95","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:22.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ef0447bc-9a75-45ad-8dca-b64d1252cb95","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:22.587Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ef0447bc-9a75-45ad-8dca-b64d1252cb95"}
{"level":"info","ts":"2026-06-10T10:30:22.587Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ef0447bc-9a75-45ad-8dca-b64d1252cb95","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:25.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ca1c1b31-5fe8-470a-ad66-f46b74385668","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:25.588Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ca1c1b31-5fe8-470a-ad66-f46b74385668","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:25.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ca1c1b31-5fe8-470a-ad66-f46b74385668","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:25.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ca1c1b31-5fe8-470a-ad66-f46b74385668"}
{"level":"info","ts":"2026-06-10T10:30:25.588Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"ca1c1b31-5fe8-470a-ad66-f46b74385668","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:28.589Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85855f52-efd5-4e05-a768-d4ff14f68882","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:28.589Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85855f52-efd5-4e05-a768-d4ff14f68882","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:28.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85855f52-efd5-4e05-a768-d4ff14f68882","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:28.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85855f52-efd5-4e05-a768-d4ff14f68882"}
{"level":"info","ts":"2026-06-10T10:30:28.589Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"85855f52-efd5-4e05-a768-d4ff14f68882","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:31.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f2caaa7b-e149-4d0a-899f-883001b2937b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:31.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f2caaa7b-e149-4d0a-899f-883001b2937b","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:31.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f2caaa7b-e149-4d0a-899f-883001b2937b","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:31.590Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f2caaa7b-e149-4d0a-899f-883001b2937b"}
{"level":"info","ts":"2026-06-10T10:30:31.590Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f2caaa7b-e149-4d0a-899f-883001b2937b","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:34.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f90923b4-ef85-4003-9eea-b8d51cdee91e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:34.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f90923b4-ef85-4003-9eea-b8d51cdee91e","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:34.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f90923b4-ef85-4003-9eea-b8d51cdee91e","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:34.590Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f90923b4-ef85-4003-9eea-b8d51cdee91e"}
{"level":"info","ts":"2026-06-10T10:30:34.590Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"f90923b4-ef85-4003-9eea-b8d51cdee91e","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:37.591Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"88c54b60-bbf8-46f1-9d2a-b42e43b15754","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:37.591Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"88c54b60-bbf8-46f1-9d2a-b42e43b15754","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:37.591Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"88c54b60-bbf8-46f1-9d2a-b42e43b15754","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:37.591Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"88c54b60-bbf8-46f1-9d2a-b42e43b15754"}
{"level":"info","ts":"2026-06-10T10:30:37.591Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"88c54b60-bbf8-46f1-9d2a-b42e43b15754","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:40.591Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4b16800b-c92a-4fef-a1f1-5b4de2e9f061","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:40.591Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4b16800b-c92a-4fef-a1f1-5b4de2e9f061","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:40.592Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4b16800b-c92a-4fef-a1f1-5b4de2e9f061","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:40.592Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4b16800b-c92a-4fef-a1f1-5b4de2e9f061"}
{"level":"info","ts":"2026-06-10T10:30:40.592Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4b16800b-c92a-4fef-a1f1-5b4de2e9f061","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:43.593Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d80017ac-40a9-4caa-bdb9-5491ec0f977a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:43.593Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d80017ac-40a9-4caa-bdb9-5491ec0f977a","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:43.593Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d80017ac-40a9-4caa-bdb9-5491ec0f977a","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:43.593Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d80017ac-40a9-4caa-bdb9-5491ec0f977a"}
{"level":"info","ts":"2026-06-10T10:30:43.593Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d80017ac-40a9-4caa-bdb9-5491ec0f977a","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:46.594Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da29f691-32c4-44b6-a9f9-524e30887a04","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:46.594Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da29f691-32c4-44b6-a9f9-524e30887a04","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:46.594Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da29f691-32c4-44b6-a9f9-524e30887a04","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:46.594Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da29f691-32c4-44b6-a9f9-524e30887a04"}
{"level":"info","ts":"2026-06-10T10:30:46.594Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da29f691-32c4-44b6-a9f9-524e30887a04","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:46.906Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"42a12e52-6803-4661-93f6-f0c9d9a43756","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:30:46.907Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"42a12e52-6803-4661-93f6-f0c9d9a43756","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:30:46.907Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"42a12e52-6803-4661-93f6-f0c9d9a43756","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:30:46.907Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"42a12e52-6803-4661-93f6-f0c9d9a43756","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:30:49.595Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7628b242-6f78-4e88-aa9d-cc1e312debbd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:49.595Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7628b242-6f78-4e88-aa9d-cc1e312debbd","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:49.595Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7628b242-6f78-4e88-aa9d-cc1e312debbd","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:49.595Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7628b242-6f78-4e88-aa9d-cc1e312debbd"}
{"level":"info","ts":"2026-06-10T10:30:49.595Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7628b242-6f78-4e88-aa9d-cc1e312debbd","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:49.743Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"d6b864c2-3360-4c4f-8933-a74ca87eac92","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:49.743Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"d6b864c2-3360-4c4f-8933-a74ca87eac92","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:30:49.743Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"d6b864c2-3360-4c4f-8933-a74ca87eac92","pod":"fail-hxn94-small-group-worker-s48tx","ip":"10.132.0.42"}
{"level":"info","ts":"2026-06-10T10:30:49.743Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"d6b864c2-3360-4c4f-8933-a74ca87eac92","rayCluster":"fail-hxn94","podIPs":["10.134.0.17","10.132.0.42"]}
{"level":"info","ts":"2026-06-10T10:30:52.116Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5"}
{"level":"info","ts":"2026-06-10T10:30:52.116Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5"}
{"level":"info","ts":"2026-06-10T10:30:52.116Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:30:52.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:30:52.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:30:52.116Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:30:52.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:30:52.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:30:52.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:30:52.117Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:30:52.117Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:30:52.117Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:30:52.117Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:30:52.117Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7bd5c71d-a427-498c-860b-89244cc97ec5","seconds":300}
{"level":"info","ts":"2026-06-10T10:30:52.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4e12eea5-8f59-4124-b3a1-dc5ca5f39c4d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:52.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4e12eea5-8f59-4124-b3a1-dc5ca5f39c4d","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:52.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4e12eea5-8f59-4124-b3a1-dc5ca5f39c4d","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:52.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4e12eea5-8f59-4124-b3a1-dc5ca5f39c4d"}
{"level":"info","ts":"2026-06-10T10:30:52.597Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4e12eea5-8f59-4124-b3a1-dc5ca5f39c4d","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358"}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358"}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","podName":"fail-hxn94-head-nj5fr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:29:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","podName":"fail-hxn94-head-nj5fr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:29:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:30:53.123Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4a583f31-f989-40ef-b048-d3a1828db358","name":"fail-hxn94","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:29:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:30:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"e30f95e0-d6ba-4fd3-8623-92e0a918ae9f","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f8b9075d-bdbd-48b7-bfdd-5efbe30df7da","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"cc9c5969-ffb7-46e7-a75e-ad12bdc75efd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"cc9c5969-ffb7-46e7-a75e-ad12bdc75efd","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"cc9c5969-ffb7-46e7-a75e-ad12bdc75efd","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f8b9075d-bdbd-48b7-bfdd-5efbe30df7da","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f8b9075d-bdbd-48b7-bfdd-5efbe30df7da","pod":"fail-hxn94-small-group-worker-s48tx","ip":"10.132.0.42"}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"f8b9075d-bdbd-48b7-bfdd-5efbe30df7da","rayCluster":"fail-hxn94","podIPs":["10.134.0.17","10.132.0.42"]}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"cc9c5969-ffb7-46e7-a75e-ad12bdc75efd"}
{"level":"info","ts":"2026-06-10T10:30:53.137Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"cc9c5969-ffb7-46e7-a75e-ad12bdc75efd","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:53.156Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"e30f95e0-d6ba-4fd3-8623-92e0a918ae9f","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:30:53.168Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"e30f95e0-d6ba-4fd3-8623-92e0a918ae9f","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:30:53.168Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"e30f95e0-d6ba-4fd3-8623-92e0a918ae9f","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417"}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417"}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","podName":"fail-hxn94-head-nj5fr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:29:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","podName":"fail-hxn94-head-nj5fr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:29:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:30:55.138Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"95707440-5fee-4b1b-9f07-80723b25f417","seconds":300}
{"level":"info","ts":"2026-06-10T10:30:55.598Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bbdd5e73-a708-4bb4-9b2f-dfc88fe64579","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:55.598Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bbdd5e73-a708-4bb4-9b2f-dfc88fe64579","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:55.598Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bbdd5e73-a708-4bb4-9b2f-dfc88fe64579","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:55.598Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bbdd5e73-a708-4bb4-9b2f-dfc88fe64579"}
{"level":"info","ts":"2026-06-10T10:30:55.598Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bbdd5e73-a708-4bb4-9b2f-dfc88fe64579","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:30:58.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc02ec54-afec-4a8f-bf82-0ad15c506a3c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:30:58.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc02ec54-afec-4a8f-bf82-0ad15c506a3c","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:30:58.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc02ec54-afec-4a8f-bf82-0ad15c506a3c","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:30:58.599Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc02ec54-afec-4a8f-bf82-0ad15c506a3c"}
{"level":"info","ts":"2026-06-10T10:30:58.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc02ec54-afec-4a8f-bf82-0ad15c506a3c","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:31:01.601Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"02b5386d-e4b6-425d-8ca1-b4a96e56a8da","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:01.601Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"02b5386d-e4b6-425d-8ca1-b4a96e56a8da","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:01.601Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"02b5386d-e4b6-425d-8ca1-b4a96e56a8da","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:01.601Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"02b5386d-e4b6-425d-8ca1-b4a96e56a8da"}
{"level":"info","ts":"2026-06-10T10:31:01.601Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"02b5386d-e4b6-425d-8ca1-b4a96e56a8da","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58"}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58"}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:31:04.150Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b2ad667-bf05-4950-bf2d-144d50839c58","name":"fail-hxn94","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:30:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"abf5d117-4f5d-47ad-a1cb-ea922e844dd1","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"807c0b2a-e22d-4a03-a739-4d872b470482","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"abf5d117-4f5d-47ad-a1cb-ea922e844dd1","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"abf5d117-4f5d-47ad-a1cb-ea922e844dd1","pod":"fail-hxn94-small-group-worker-s48tx","ip":"10.132.0.42"}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"abf5d117-4f5d-47ad-a1cb-ea922e844dd1","rayCluster":"fail-hxn94","podIPs":["10.134.0.17","10.132.0.42"]}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d7e3484b-abe8-4683-833d-82d457bc99c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d7e3484b-abe8-4683-833d-82d457bc99c9","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d7e3484b-abe8-4683-833d-82d457bc99c9","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d7e3484b-abe8-4683-833d-82d457bc99c9"}
{"level":"info","ts":"2026-06-10T10:31:04.163Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d7e3484b-abe8-4683-833d-82d457bc99c9","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:31:04.172Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"807c0b2a-e22d-4a03-a739-4d872b470482","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:31:04.181Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"807c0b2a-e22d-4a03-a739-4d872b470482","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:31:04.181Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"807c0b2a-e22d-4a03-a739-4d872b470482","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:04.602Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30c9e304-5193-4dcc-b890-fdc8d3a02ded","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:04.602Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30c9e304-5193-4dcc-b890-fdc8d3a02ded","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:04.602Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30c9e304-5193-4dcc-b890-fdc8d3a02ded","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:04.602Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30c9e304-5193-4dcc-b890-fdc8d3a02ded"}
{"level":"info","ts":"2026-06-10T10:31:04.602Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"30c9e304-5193-4dcc-b890-fdc8d3a02ded","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c"}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c"}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:31:06.164Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"585d65ce-b650-4708-955d-ad76376dec4c","seconds":300}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65"}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65"}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:31:07.011Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"a075b79e-2719-424d-aaa1-b56118c95e65","seconds":300}
{"level":"info","ts":"2026-06-10T10:31:07.603Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc584c8f-ddd6-4e09-b032-9dd9b8a1c858","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:07.603Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc584c8f-ddd6-4e09-b032-9dd9b8a1c858","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:07.603Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc584c8f-ddd6-4e09-b032-9dd9b8a1c858","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:07.603Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc584c8f-ddd6-4e09-b032-9dd9b8a1c858"}
{"level":"info","ts":"2026-06-10T10:31:07.603Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"bc584c8f-ddd6-4e09-b032-9dd9b8a1c858","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c"}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c"}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:29:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:29:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:31:08.008Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c5a0e593-f1ea-496c-8cd4-919b874a0d8c","name":"fail-hxn94","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3390a940-13e5-4102-8f70-edb4d3bf2925","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7d03eb23-4c3d-4032-a385-9e0c7cd567d6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7d03eb23-4c3d-4032-a385-9e0c7cd567d6","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7d03eb23-4c3d-4032-a385-9e0c7cd567d6","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"13e447b9-bd82-4729-bfe5-72c6b9b04cff","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3390a940-13e5-4102-8f70-edb4d3bf2925","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3390a940-13e5-4102-8f70-edb4d3bf2925","pod":"fail-hxn94-small-group-worker-s48tx","ip":"10.132.0.42"}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7d03eb23-4c3d-4032-a385-9e0c7cd567d6"}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"7d03eb23-4c3d-4032-a385-9e0c7cd567d6","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:31:08.021Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3390a940-13e5-4102-8f70-edb4d3bf2925","rayCluster":"fail-hxn94","podIPs":["10.134.0.17","10.132.0.42"]}
{"level":"info","ts":"2026-06-10T10:31:08.031Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"13e447b9-bd82-4729-bfe5-72c6b9b04cff","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:31:08.040Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"13e447b9-bd82-4729-bfe5-72c6b9b04cff","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:31:08.040Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"13e447b9-bd82-4729-bfe5-72c6b9b04cff","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f"}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f"}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:29:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:29:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:31:10.022Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"1a267562-ecac-4bce-804b-fd5950ead21f","seconds":300}
{"level":"info","ts":"2026-06-10T10:31:10.604Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d6d7562-a75e-4f47-961c-f160fe47b7ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:10.604Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d6d7562-a75e-4f47-961c-f160fe47b7ff","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:10.604Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d6d7562-a75e-4f47-961c-f160fe47b7ff","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:10.604Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d6d7562-a75e-4f47-961c-f160fe47b7ff"}
{"level":"info","ts":"2026-06-10T10:31:10.604Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"0d6d7562-a75e-4f47-961c-f160fe47b7ff","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:31:13.605Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"935ea1db-b7cd-4466-8117-7bdd5a7190ad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:13.605Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"935ea1db-b7cd-4466-8117-7bdd5a7190ad","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:13.605Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"935ea1db-b7cd-4466-8117-7bdd5a7190ad","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:13.605Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"935ea1db-b7cd-4466-8117-7bdd5a7190ad"}
{"level":"info","ts":"2026-06-10T10:31:13.605Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"935ea1db-b7cd-4466-8117-7bdd5a7190ad","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:31:16.606Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"c155d513-a6a3-4646-9db2-256dd9604126","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:16.606Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"c155d513-a6a3-4646-9db2-256dd9604126","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:16.606Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"c155d513-a6a3-4646-9db2-256dd9604126","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:16.606Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"c155d513-a6a3-4646-9db2-256dd9604126"}
{"level":"info","ts":"2026-06-10T10:31:16.606Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"c155d513-a6a3-4646-9db2-256dd9604126","RayCluster":"fail-hxn94","State":""}
{"level":"info","ts":"2026-06-10T10:31:19.032Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad"}
{"level":"info","ts":"2026-06-10T10:31:19.032Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad"}
{"level":"info","ts":"2026-06-10T10:31:19.032Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:19.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:19.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:19.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:19.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:19.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:19.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:19.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:19.033Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:31:19.033Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c68c0f67-91d8-403d-b53e-b530ce7da6ad","name":"fail-hxn94","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:19Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:19.045Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"bea0b71e-36df-402b-a514-29c8eff1e6db","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:19.045Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"46c9d06d-b4af-4774-8c9a-21f223c81a6b","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:19.045Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"bea0b71e-36df-402b-a514-29c8eff1e6db","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"bea0b71e-36df-402b-a514-29c8eff1e6db","pod":"fail-hxn94-small-group-worker-s48tx","ip":"10.132.0.42"}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"bea0b71e-36df-402b-a514-29c8eff1e6db","rayCluster":"fail-hxn94","podIPs":["10.134.0.17","10.132.0.42"]}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"041326f5-a102-428d-a20f-a02c65b74f1b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"041326f5-a102-428d-a20f-a02c65b74f1b","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"041326f5-a102-428d-a20f-a02c65b74f1b","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"041326f5-a102-428d-a20f-a02c65b74f1b"}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"041326f5-a102-428d-a20f-a02c65b74f1b","head service name":"fail-hxn94-head-svc","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"041326f5-a102-428d-a20f-a02c65b74f1b","head service URL":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"041326f5-a102-428d-a20f-a02c65b74f1b","SubmissionMode":"HTTPMode","RayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"041326f5-a102-428d-a20f-a02c65b74f1b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-68dpl","rayClusterName":"fail-hxn94","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:29:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-68dpl","rayClusterName":"fail-hxn94","dashboardURL":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:29:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:31:19.046Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"041326f5-a102-428d-a20f-a02c65b74f1b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:31:19.056Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"39c05088-b0d6-45b9-afe7-df1916371ce7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:19.056Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"39c05088-b0d6-45b9-afe7-df1916371ce7","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:19.056Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"39c05088-b0d6-45b9-afe7-df1916371ce7","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:19.056Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"39c05088-b0d6-45b9-afe7-df1916371ce7"}
{"level":"info","ts":"2026-06-10T10:31:19.061Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"46c9d06d-b4af-4774-8c9a-21f223c81a6b","name":"fail-hxn94-head"}
{"level":"info","ts":"2026-06-10T10:31:19.067Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"39c05088-b0d6-45b9-afe7-df1916371ce7","JobId":"fail-68dpl"}
{"level":"info","ts":"2026-06-10T10:31:19.067Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"39c05088-b0d6-45b9-afe7-df1916371ce7","rayJob":"fail","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/fail.py\",\"submission_id\":\"fail-68dpl\"}"}
{"level":"info","ts":"2026-06-10T10:31:19.077Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"46c9d06d-b4af-4774-8c9a-21f223c81a6b","name":"fail-hxn94-workers"}
{"level":"info","ts":"2026-06-10T10:31:19.077Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"46c9d06d-b4af-4774-8c9a-21f223c81a6b","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:19.919Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"92e8ad42-f8d2-4116-8253-9aa719dc366e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:19.919Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"92e8ad42-f8d2-4116-8253-9aa719dc366e","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:19.919Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"92e8ad42-f8d2-4116-8253-9aa719dc366e","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:19.919Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"92e8ad42-f8d2-4116-8253-9aa719dc366e"}
{"level":"info","ts":"2026-06-10T10:31:19.922Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"92e8ad42-f8d2-4116-8253-9aa719dc366e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-68dpl","rayClusterName":"fail-hxn94","dashboardURL":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:29:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:31:19Z"},"jobId":"fail-68dpl","rayClusterName":"fail-hxn94","dashboardURL":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:29:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:19Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:31:19.922Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"92e8ad42-f8d2-4116-8253-9aa719dc366e","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:31:19.935Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"2becc61d-7df1-4336-96e8-242fa8b5076c","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:19.935Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"2becc61d-7df1-4336-96e8-242fa8b5076c","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:19.935Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"2becc61d-7df1-4336-96e8-242fa8b5076c","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:19.935Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"2becc61d-7df1-4336-96e8-242fa8b5076c"}
{"level":"info","ts":"2026-06-10T10:31:19.937Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"2becc61d-7df1-4336-96e8-242fa8b5076c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:31:19Z"},"jobId":"fail-68dpl","rayClusterName":"fail-hxn94","dashboardURL":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:29:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:19Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:31:19Z"},"jobId":"fail-68dpl","rayClusterName":"fail-hxn94","dashboardURL":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:29:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:19Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b"}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b"}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:21.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:21.047Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:31:21.047Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b7a8d74c-7ab5-4bdf-ab83-9fea2942e42b","seconds":300}
{"level":"info","ts":"2026-06-10T10:31:22.920Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da65bd2e-41a6-4c22-94ac-27e66f42b403","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:22.920Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da65bd2e-41a6-4c22-94ac-27e66f42b403","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:22.920Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da65bd2e-41a6-4c22-94ac-27e66f42b403","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:22.920Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da65bd2e-41a6-4c22-94ac-27e66f42b403"}
{"level":"info","ts":"2026-06-10T10:31:22.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da65bd2e-41a6-4c22-94ac-27e66f42b403","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:31:19Z"},"jobId":"fail-68dpl","rayClusterName":"fail-hxn94","dashboardURL":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:29:19Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:19Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:31:19Z","endTime":"2026-06-10T10:31:21Z"},"jobId":"fail-68dpl","rayClusterName":"fail-hxn94","dashboardURL":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:31:19,619\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-68dpl: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:29:19Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:19Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:31:22.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"da65bd2e-41a6-4c22-94ac-27e66f42b403","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-10T10:31:22.938Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"1be0b888-c4cd-4e5c-8378-5298028186d4","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:22.938Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"1be0b888-c4cd-4e5c-8378-5298028186d4","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:31:22 +0000 UTC","Now":"2026-06-10T10:31:22.938Z","ShutdownTime":"2026-06-10T10:31:22.000Z"}
{"level":"info","ts":"2026-06-10T10:31:23.272Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"de5def5f-3c17-4867-a798-633698d0541d","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:23.272Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"de5def5f-3c17-4867-a798-633698d0541d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:31:23.282Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"de5def5f-3c17-4867-a798-633698d0541d"}
{"level":"info","ts":"2026-06-10T10:31:23.282Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"de5def5f-3c17-4867-a798-633698d0541d","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:31:23.282Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"de5def5f-3c17-4867-a798-633698d0541d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:31:23.282Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"de5def5f-3c17-4867-a798-633698d0541d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:31:23.292Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6dcd34b6-3d1b-4353-a2df-3b1593f16805","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:23.292Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6dcd34b6-3d1b-4353-a2df-3b1593f16805","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.292Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6dcd34b6-3d1b-4353-a2df-3b1593f16805","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.292Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6dcd34b6-3d1b-4353-a2df-3b1593f16805","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.309Z","logger":"raycluster-resource","msg":"default","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.309Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-4bp9q","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:31:23.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6dcd34b6-3d1b-4353-a2df-3b1593f16805","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72","namespacedName":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6dcd34b6-3d1b-4353-a2df-3b1593f16805"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6dcd34b6-3d1b-4353-a2df-3b1593f16805","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72","mode":"IntegratedOAuth","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"820fbb6b-5810-46bf-a80a-d0706a426c55","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08be0f0a-ab02-4b6a-ba40-af9dc5ed8e73","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08be0f0a-ab02-4b6a-ba40-af9dc5ed8e73","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08be0f0a-ab02-4b6a-ba40-af9dc5ed8e73","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08be0f0a-ab02-4b6a-ba40-af9dc5ed8e73"}
{"level":"info","ts":"2026-06-10T10:31:23.319Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08be0f0a-ab02-4b6a-ba40-af9dc5ed8e73","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:23.399Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.399Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.399Z","logger":"raycluster-resource","msg":"default","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.399Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-4bp9q","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:31:23.399Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","name":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:23.399Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","error":"head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"error","ts":"2026-06-10T10:31:23.400Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.400Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"info","ts":"2026-06-10T10:31:23.400Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.400Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.400Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.400Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:31:23.400Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","name":"stop-4bp9q","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:23.404Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"820fbb6b-5810-46bf-a80a-d0706a426c55","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.405Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"c1ab8407-64af-48e1-a955-6bf71fd6e474","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8de12a42-d838-482d-9fbe-a2bb5379eb87","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:23.408Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8de12a42-d838-482d-9fbe-a2bb5379eb87","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8de12a42-d838-482d-9fbe-a2bb5379eb87","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.409Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8de12a42-d838-482d-9fbe-a2bb5379eb87"}
{"level":"info","ts":"2026-06-10T10:31:23.409Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8de12a42-d838-482d-9fbe-a2bb5379eb87","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:23.411Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da"}
{"level":"info","ts":"2026-06-10T10:31:23.411Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","rayCluster":"stop-4bp9q","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:31:23.411Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72","name":"stop-4bp9q-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","name":"stop-4bp9q","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-4bp9q\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"stop-4bp9q"}}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a"}
{"level":"error","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89c3b316-5c08-47dd-9e39-bebf7a8d1c0a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6"}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6"}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","error":"head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"error","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.413Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.413Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:31:23.413Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","name":"stop-4bp9q","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:23.418Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"c1ab8407-64af-48e1-a955-6bf71fd6e474","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:31:23.422Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72","name":"test-ns-bbzgk-stop-4bp9q","operation":"created"}
{"level":"info","ts":"2026-06-10T10:31:23.422Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.499Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da"}
{"level":"info","ts":"2026-06-10T10:31:23.499Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72","name":"kube-rbac-proxy-config-stop-4bp9q","operation":"created"}
{"level":"info","ts":"2026-06-10T10:31:23.499Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","rayCluster":"stop-4bp9q","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6"}
{"level":"error","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f908d7a-2732-4260-9c31-e3ea5a0290b6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"116ad304-6cdb-4dee-9e41-e31de51ed76c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"116ad304-6cdb-4dee-9e41-e31de51ed76c","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"116ad304-6cdb-4dee-9e41-e31de51ed76c","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22","error":"head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"error","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"116ad304-6cdb-4dee-9e41-e31de51ed76c"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22"}
{"level":"info","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"116ad304-6cdb-4dee-9e41-e31de51ed76c","RayCluster":"stop-4bp9q","State":""}
{"level":"error","ts":"2026-06-10T10:31:23.500Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dd8d40b2-ae7c-499d-8c46-3c342e3bcf22","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.503Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"c1ab8407-64af-48e1-a955-6bf71fd6e474","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:31:23.503Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"c1ab8407-64af-48e1-a955-6bf71fd6e474","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.503Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"708e6d8b-e439-4443-bb79-ee4c88022c7e","cluster":"stop-4bp9q"}
{"level":"error","ts":"2026-06-10T10:31:23.505Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-4bp9q\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.505Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72"}
{"level":"error","ts":"2026-06-10T10:31:23.505Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"15cade37-93b7-4168-b78c-54f7909e9a72","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-4bp9q\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.505Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"dcc7bb07-e013-4770-9b69-43fc79251afa","namespacedName":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.505Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"dcc7bb07-e013-4770-9b69-43fc79251afa","mode":"IntegratedOAuth","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.505Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"dcc7bb07-e013-4770-9b69-43fc79251afa","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.507Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da"}
{"level":"info","ts":"2026-06-10T10:31:23.507Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","rayCluster":"stop-4bp9q","podIPs":[]}
{"level":"error","ts":"2026-06-10T10:31:23.507Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-4bp9q\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.507Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da"}
{"level":"error","ts":"2026-06-10T10:31:23.507Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ab96f879-4cee-4ca2-8313-371f270784da","error":"Certificate.cert-manager.io \"ray-worker-cert-stop-4bp9q\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.508Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"cd21d177-df8f-498e-91e5-5b4bffc074b0","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b"}
{"level":"info","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b"}
{"level":"info","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b","error":"head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"error","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"info","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b"}
{"level":"error","ts":"2026-06-10T10:31:23.510Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"edb0716d-6552-415f-8cb9-68541405367b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a","error":"head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"error","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"cd21d177-df8f-498e-91e5-5b4bffc074b0"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"error","ts":"2026-06-10T10:31:23.599Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"cd21d177-df8f-498e-91e5-5b4bffc074b0","error":"certificates.cert-manager.io \"ray-worker-cert-stop-4bp9q\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:31:23.599Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"cd21d177-df8f-498e-91e5-5b4bffc074b0","error":"certificates.cert-manager.io \"ray-worker-cert-stop-4bp9q\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"cd21d177-df8f-498e-91e5-5b4bffc074b0"}
{"level":"error","ts":"2026-06-10T10:31:23.599Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"cd21d177-df8f-498e-91e5-5b4bffc074b0","error":"certificates.cert-manager.io \"ray-worker-cert-stop-4bp9q\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a"}
{"level":"error","ts":"2026-06-10T10:31:23.599Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f299aa94-8703-4882-8f89-f5498e26385a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"c5cf8a63-1ef6-4ec7-8ab6-25bb1a03c5b9","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.599Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"dcc7bb07-e013-4770-9b69-43fc79251afa","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.600Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"c5cf8a63-1ef6-4ec7-8ab6-25bb1a03c5b9"}
{"level":"info","ts":"2026-06-10T10:31:23.600Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"0cdae54e-23e0-419d-9c43-b317e4f0f435","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.600Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"1b3301d3-1418-49f8-8666-90227d3cf746","namespacedName":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.600Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"0cdae54e-23e0-419d-9c43-b317e4f0f435"}
{"level":"info","ts":"2026-06-10T10:31:23.600Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"1b3301d3-1418-49f8-8666-90227d3cf746","mode":"IntegratedOAuth","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.600Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"1b3301d3-1418-49f8-8666-90227d3cf746","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.603Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"708e6d8b-e439-4443-bb79-ee4c88022c7e","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:31:23.610Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"de6860f1-8a7b-47a3-9875-b9b6a1e61bb5","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.611Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"de6860f1-8a7b-47a3-9875-b9b6a1e61bb5"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"708e6d8b-e439-4443-bb79-ee4c88022c7e","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"708e6d8b-e439-4443-bb79-ee4c88022c7e","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ddf7b6a5-caf8-4d86-a3a2-375c70c169b7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ddf7b6a5-caf8-4d86-a3a2-375c70c169b7","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ddf7b6a5-caf8-4d86-a3a2-375c70c169b7","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"8fd322e2-5fcf-4e01-bf4d-a1a1edd7f010","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ddf7b6a5-caf8-4d86-a3a2-375c70c169b7"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ddf7b6a5-caf8-4d86-a3a2-375c70c169b7","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220","error":"head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"error","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220"}
{"level":"info","ts":"2026-06-10T10:31:23.699Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"1b3301d3-1418-49f8-8666-90227d3cf746","cluster":"stop-4bp9q"}
{"level":"error","ts":"2026-06-10T10:31:23.699Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"07aee863-73dc-4177-abbe-eb36d9f74220","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.710Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"8fd322e2-5fcf-4e01-bf4d-a1a1edd7f010","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:31:23.718Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"8fd322e2-5fcf-4e01-bf4d-a1a1edd7f010","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:31:23.718Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"8fd322e2-5fcf-4e01-bf4d-a1a1edd7f010","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5"}
{"level":"info","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5"}
{"level":"info","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5","error":"head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"error","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"info","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5"}
{"level":"error","ts":"2026-06-10T10:31:23.860Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"09c6649d-bfe3-403f-8a71-3ec98be224a5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e"}
{"level":"info","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e"}
{"level":"info","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e","error":"head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"error","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found"}
{"level":"info","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"stop-4bp9q","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e"}
{"level":"error","ts":"2026-06-10T10:31:24.181Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"e787b845-671c-411d-a83e-791f8a44760e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-4bp9q not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8"}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8"}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8"}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8"}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","mode":"IntegratedOAuth","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","cluster":"stop-4bp9q","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","cluster":"stop-4bp9q","authType":"IntegratedOAuth","serviceAccount":"stop-4bp9q-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-4bp9q","ray.io/group":"headgroup","ray.io/identifier":"stop-4bp9q-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:31:24.822Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","enabled":true}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","name":"stop-4bp9q-head-wdfdc"}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8"}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","nodeType":"worker","rayStartParams":{"address":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","rayStartCmd":"ray start  --address=stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:31:24.849Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","enabled":true}
{"level":"info","ts":"2026-06-10T10:31:24.863Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","name":"stop-4bp9q-small-group-worker-dp9vm"}
{"level":"info","ts":"2026-06-10T10:31:24.863Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:24.863Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-4bp9q-head-svc"},"newHead":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"}}
{"level":"info","ts":"2026-06-10T10:31:24.863Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"d309e84c-1eca-4137-9206-8e8e1e1c05a8","name":"stop-4bp9q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3"}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3"}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-4bp9q-head-svc"},"newHead":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"}}
{"level":"info","ts":"2026-06-10T10:31:24.876Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","name":"stop-4bp9q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:24.877Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"cd156bed-e203-487c-8447-d616187e8cf8","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:24.877Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"815532af-76e5-4c5a-bfc7-150c4a08b278","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:24.877Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c70db3d-3a75-45d3-84b4-1b7cec085582","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:24.877Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c70db3d-3a75-45d3-84b4-1b7cec085582","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:24.877Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c70db3d-3a75-45d3-84b4-1b7cec085582","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:24.877Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"cd156bed-e203-487c-8447-d616187e8cf8"}
{"level":"info","ts":"2026-06-10T10:31:24.877Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c70db3d-3a75-45d3-84b4-1b7cec085582"}
{"level":"info","ts":"2026-06-10T10:31:24.877Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c70db3d-3a75-45d3-84b4-1b7cec085582","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","name":"stop-4bp9q","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-4bp9q\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"stop-4bp9q"}}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3"}
{"level":"error","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"ef7b1363-e2f6-4191-8f21-7edf4c6121b3","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-4bp9q\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f"}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f"}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:31:24.899Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9eaee4f8-230c-483c-90e7-0168946f4d4f","name":"stop-4bp9q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:24.906Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"815532af-76e5-4c5a-bfc7-150c4a08b278","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"4bd177b5-49bb-468a-8e5e-d6cb617d7e65","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"62323bee-cf7d-457a-8696-1d5329d258b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"62323bee-cf7d-457a-8696-1d5329d258b1","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"4bd177b5-49bb-468a-8e5e-d6cb617d7e65"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"62323bee-cf7d-457a-8696-1d5329d258b1","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.910Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:24.911Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:24.911Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"62323bee-cf7d-457a-8696-1d5329d258b1"}
{"level":"info","ts":"2026-06-10T10:31:24.911Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"62323bee-cf7d-457a-8696-1d5329d258b1","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:24.911Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:31:24.911Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","name":"stop-4bp9q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:24.917Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","name":"stop-4bp9q","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-4bp9q\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-bbzgk","name":"stop-4bp9q"}}
{"level":"info","ts":"2026-06-10T10:31:24.917Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b"}
{"level":"error","ts":"2026-06-10T10:31:24.917Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"73ba8d7a-2f9d-4efa-ad91-99435ffa565b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-4bp9q\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:31:24.917Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"815532af-76e5-4c5a-bfc7-150c4a08b278","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:31:24.917Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"815532af-76e5-4c5a-bfc7-150c4a08b278","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:24.917Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"adc84f8f-458b-465b-8251-56039972b4c3","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644"}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644"}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:31:24.922Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3753bf2b-3cb0-4a6d-b0fb-b19741004644","seconds":300}
{"level":"info","ts":"2026-06-10T10:31:24.929Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"adc84f8f-458b-465b-8251-56039972b4c3","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:31:24.939Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"adc84f8f-458b-465b-8251-56039972b4c3","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:31:24.939Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"adc84f8f-458b-465b-8251-56039972b4c3","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e"}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e"}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","podName":"stop-4bp9q-small-group-worker-dp9vm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","podName":"stop-4bp9q-small-group-worker-dp9vm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:31:25.299Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"4cc4a5a5-365c-4937-a328-cb9a83ff205e","seconds":300}
{"level":"info","ts":"2026-06-10T10:31:25.318Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6"}
{"level":"info","ts":"2026-06-10T10:31:25.318Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6"}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:31:25.319Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"9ded1e3e-e408-4612-be73-880b082f6ea6","seconds":300}
{"level":"info","ts":"2026-06-10T10:31:25.608Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590"}
{"level":"info","ts":"2026-06-10T10:31:25.608Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590"}
{"level":"info","ts":"2026-06-10T10:31:25.608Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:25.608Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:31:25.609Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"c556dbb3-fa6a-4f56-987b-316270ada590","seconds":300}
{"level":"info","ts":"2026-06-10T10:31:25.938Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4f333032-6a48-430f-9e17-c34a9a395a8d","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:25.938Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"4f333032-6a48-430f-9e17-c34a9a395a8d","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:31:22 +0000 UTC","Now":"2026-06-10T10:31:25.938Z","ShutdownTime":"2026-06-10T10:31:22.000Z"}
{"level":"info","ts":"2026-06-10T10:31:26.238Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499"}
{"level":"info","ts":"2026-06-10T10:31:26.238Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499"}
{"level":"info","ts":"2026-06-10T10:31:26.238Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:26.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:26.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:26.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:26.239Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:26.239Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:26.239Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:26.239Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:26.239Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:26.239Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:26.239Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"newHead":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"}}
{"level":"info","ts":"2026-06-10T10:31:26.239Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"cf775787-98fb-47bb-8f7b-159694010499","name":"stop-4bp9q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:31:26.250Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e34182cc-9466-44e1-ab29-5490324cfa8c","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:26.250Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"955302f9-552e-402f-aa8e-570c3f251909","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:26.251Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e34182cc-9466-44e1-ab29-5490324cfa8c","pod":"stop-4bp9q-head-wdfdc","ip":"10.134.0.18"}
{"level":"info","ts":"2026-06-10T10:31:26.251Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c139adc-1476-46f1-886b-9b8720e56702","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:26.251Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c139adc-1476-46f1-886b-9b8720e56702","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:26.251Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e34182cc-9466-44e1-ab29-5490324cfa8c","pod":"stop-4bp9q-small-group-worker-dp9vm","ip":"10.133.0.14"}
{"level":"info","ts":"2026-06-10T10:31:26.251Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c139adc-1476-46f1-886b-9b8720e56702","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:26.251Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c139adc-1476-46f1-886b-9b8720e56702"}
{"level":"info","ts":"2026-06-10T10:31:26.251Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c139adc-1476-46f1-886b-9b8720e56702","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:26.262Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"955302f9-552e-402f-aa8e-570c3f251909","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:31:26.262Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e34182cc-9466-44e1-ab29-5490324cfa8c","certificate":"ray-head-cert-stop-4bp9q","dnsNames":["stop-4bp9q-head-svc","localhost","stop-4bp9q-head-svc.test-ns-bbzgk.svc","stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.18","10.133.0.14","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:31:26.272Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"955302f9-552e-402f-aa8e-570c3f251909","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:31:26.272Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"955302f9-552e-402f-aa8e-570c3f251909","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:26.273Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e34182cc-9466-44e1-ab29-5490324cfa8c","certificate":"ray-worker-cert-stop-4bp9q","dnsNames":["stop-4bp9q-worker-svc","localhost","stop-4bp9q-small-group","stop-4bp9q-worker-svc.test-ns-bbzgk.svc","stop-4bp9q-worker-svc.test-ns-bbzgk.svc.cluster.local","*.stop-4bp9q-worker-svc.test-ns-bbzgk.svc","*.stop-4bp9q-worker-svc.test-ns-bbzgk.svc.cluster.local","*-worker-*.test-ns-bbzgk.svc","*-worker-*.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.18","10.133.0.14","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:31:26.273Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e34182cc-9466-44e1-ab29-5490324cfa8c","rayCluster":"stop-4bp9q","podIPs":["10.134.0.18","10.133.0.14"]}
{"level":"info","ts":"2026-06-10T10:31:26.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"11bed839-ab12-450e-b93c-c1c7db95d4ea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:26.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"11bed839-ab12-450e-b93c-c1c7db95d4ea","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:26.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"11bed839-ab12-450e-b93c-c1c7db95d4ea","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:26.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"11bed839-ab12-450e-b93c-c1c7db95d4ea"}
{"level":"info","ts":"2026-06-10T10:31:26.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"11bed839-ab12-450e-b93c-c1c7db95d4ea","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad"}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad"}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:31:28.250Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:31:28.251Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:28.251Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:31:28.251Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:31:28.251Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"99b5a815-5e6d-47f8-bcd4-7eaeba14daad","seconds":300}
{"level":"info","ts":"2026-06-10T10:31:29.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4bab1d76-b701-4d37-a91b-9ec6c425be82","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:29.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4bab1d76-b701-4d37-a91b-9ec6c425be82","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:29.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4bab1d76-b701-4d37-a91b-9ec6c425be82","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:29.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4bab1d76-b701-4d37-a91b-9ec6c425be82"}
{"level":"info","ts":"2026-06-10T10:31:29.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4bab1d76-b701-4d37-a91b-9ec6c425be82","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:32.301Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0fe8ffe1-d675-45f1-8c46-ed95ebaefcc6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:32.301Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0fe8ffe1-d675-45f1-8c46-ed95ebaefcc6","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:32.301Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0fe8ffe1-d675-45f1-8c46-ed95ebaefcc6","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:32.301Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0fe8ffe1-d675-45f1-8c46-ed95ebaefcc6"}
{"level":"info","ts":"2026-06-10T10:31:32.301Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0fe8ffe1-d675-45f1-8c46-ed95ebaefcc6","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:35.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"b1c48110-0fe6-4d59-a036-afa735f7759b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:35.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"b1c48110-0fe6-4d59-a036-afa735f7759b","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:35.302Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"b1c48110-0fe6-4d59-a036-afa735f7759b","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:35.302Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"b1c48110-0fe6-4d59-a036-afa735f7759b"}
{"level":"info","ts":"2026-06-10T10:31:35.302Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"b1c48110-0fe6-4d59-a036-afa735f7759b","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:38.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6e14b473-036f-4dcc-beda-840a35eaea55","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:38.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6e14b473-036f-4dcc-beda-840a35eaea55","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:38.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6e14b473-036f-4dcc-beda-840a35eaea55","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:38.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6e14b473-036f-4dcc-beda-840a35eaea55"}
{"level":"info","ts":"2026-06-10T10:31:38.304Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"6e14b473-036f-4dcc-beda-840a35eaea55","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:41.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59edb825-6f88-4c37-bf8c-e3c911956835","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:41.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59edb825-6f88-4c37-bf8c-e3c911956835","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:41.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59edb825-6f88-4c37-bf8c-e3c911956835","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:41.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59edb825-6f88-4c37-bf8c-e3c911956835"}
{"level":"info","ts":"2026-06-10T10:31:41.304Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59edb825-6f88-4c37-bf8c-e3c911956835","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:44.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9f3858a2-c3b8-4d4d-9b6c-76b6cb01c4b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:44.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9f3858a2-c3b8-4d4d-9b6c-76b6cb01c4b6","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:44.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9f3858a2-c3b8-4d4d-9b6c-76b6cb01c4b6","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:44.305Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9f3858a2-c3b8-4d4d-9b6c-76b6cb01c4b6"}
{"level":"info","ts":"2026-06-10T10:31:44.305Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9f3858a2-c3b8-4d4d-9b6c-76b6cb01c4b6","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:46.908Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"30401092-a7ac-4d01-acad-4ef147b4d4d3","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:31:46.908Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"30401092-a7ac-4d01-acad-4ef147b4d4d3","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:31:46.908Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"30401092-a7ac-4d01-acad-4ef147b4d4d3","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:31:46.908Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"30401092-a7ac-4d01-acad-4ef147b4d4d3","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:31:47.306Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"364e82cf-47e7-497b-b38f-5dd08e5f88f6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:47.306Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"364e82cf-47e7-497b-b38f-5dd08e5f88f6","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:47.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"364e82cf-47e7-497b-b38f-5dd08e5f88f6","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:47.306Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"364e82cf-47e7-497b-b38f-5dd08e5f88f6"}
{"level":"info","ts":"2026-06-10T10:31:47.306Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"364e82cf-47e7-497b-b38f-5dd08e5f88f6","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:49.743Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"2c2f09ee-b109-4b4f-9cac-9b173942bb3f","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:31:49.743Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"2c2f09ee-b109-4b4f-9cac-9b173942bb3f","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:31:49.743Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"2c2f09ee-b109-4b4f-9cac-9b173942bb3f","pod":"fail-hxn94-small-group-worker-s48tx","ip":"10.132.0.42"}
{"level":"info","ts":"2026-06-10T10:31:49.743Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"2c2f09ee-b109-4b4f-9cac-9b173942bb3f","rayCluster":"fail-hxn94","podIPs":["10.134.0.17","10.132.0.42"]}
{"level":"info","ts":"2026-06-10T10:31:50.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ae546d21-fb1d-44bc-a925-22013a0a9744","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:50.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ae546d21-fb1d-44bc-a925-22013a0a9744","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:50.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ae546d21-fb1d-44bc-a925-22013a0a9744","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:50.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ae546d21-fb1d-44bc-a925-22013a0a9744"}
{"level":"info","ts":"2026-06-10T10:31:50.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"ae546d21-fb1d-44bc-a925-22013a0a9744","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:53.308Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e4a2232b-71aa-4160-9580-28a4ac53fff0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:53.308Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e4a2232b-71aa-4160-9580-28a4ac53fff0","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:53.308Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e4a2232b-71aa-4160-9580-28a4ac53fff0","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:53.308Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e4a2232b-71aa-4160-9580-28a4ac53fff0"}
{"level":"info","ts":"2026-06-10T10:31:53.308Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e4a2232b-71aa-4160-9580-28a4ac53fff0","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:53.612Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a9a1d352-f813-4daa-8cee-c38078a13f73","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:53.612Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a9a1d352-f813-4daa-8cee-c38078a13f73","pod":"stop-4bp9q-small-group-worker-dp9vm","ip":"10.133.0.14"}
{"level":"info","ts":"2026-06-10T10:31:53.612Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a9a1d352-f813-4daa-8cee-c38078a13f73","pod":"stop-4bp9q-head-wdfdc","ip":"10.134.0.18"}
{"level":"info","ts":"2026-06-10T10:31:53.612Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a9a1d352-f813-4daa-8cee-c38078a13f73","rayCluster":"stop-4bp9q","podIPs":["10.133.0.14","10.134.0.18"]}
{"level":"info","ts":"2026-06-10T10:31:56.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cd48811-749d-426c-bc0e-1bcdcb0088d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:56.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cd48811-749d-426c-bc0e-1bcdcb0088d3","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:56.309Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cd48811-749d-426c-bc0e-1bcdcb0088d3","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:56.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cd48811-749d-426c-bc0e-1bcdcb0088d3"}
{"level":"info","ts":"2026-06-10T10:31:56.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cd48811-749d-426c-bc0e-1bcdcb0088d3","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:31:59.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d7fb22ee-3934-4c53-9bec-edef4c76c0c0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:31:59.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d7fb22ee-3934-4c53-9bec-edef4c76c0c0","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:31:59.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d7fb22ee-3934-4c53-9bec-edef4c76c0c0","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:31:59.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d7fb22ee-3934-4c53-9bec-edef4c76c0c0"}
{"level":"info","ts":"2026-06-10T10:31:59.310Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d7fb22ee-3934-4c53-9bec-edef4c76c0c0","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:02.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d049e977-5582-4fc2-bb98-2d35c33825eb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:02.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d049e977-5582-4fc2-bb98-2d35c33825eb","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:02.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d049e977-5582-4fc2-bb98-2d35c33825eb","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:02.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d049e977-5582-4fc2-bb98-2d35c33825eb"}
{"level":"info","ts":"2026-06-10T10:32:02.311Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"d049e977-5582-4fc2-bb98-2d35c33825eb","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:05.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"bd5e1803-22c5-4f54-9047-e7ee849a934b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:05.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"bd5e1803-22c5-4f54-9047-e7ee849a934b","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:05.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"bd5e1803-22c5-4f54-9047-e7ee849a934b","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:05.312Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"bd5e1803-22c5-4f54-9047-e7ee849a934b"}
{"level":"info","ts":"2026-06-10T10:32:05.312Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"bd5e1803-22c5-4f54-9047-e7ee849a934b","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:08.313Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f8d26ee3-c24a-436f-914d-bcdc6e3cb94e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:08.313Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f8d26ee3-c24a-436f-914d-bcdc6e3cb94e","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:08.313Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f8d26ee3-c24a-436f-914d-bcdc6e3cb94e","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:08.313Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f8d26ee3-c24a-436f-914d-bcdc6e3cb94e"}
{"level":"info","ts":"2026-06-10T10:32:08.313Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f8d26ee3-c24a-436f-914d-bcdc6e3cb94e","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:11.314Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cec8d85-3d7e-4d93-930f-bfda6ed5959e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:11.314Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cec8d85-3d7e-4d93-930f-bfda6ed5959e","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:11.314Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cec8d85-3d7e-4d93-930f-bfda6ed5959e","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:11.314Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cec8d85-3d7e-4d93-930f-bfda6ed5959e"}
{"level":"info","ts":"2026-06-10T10:32:11.314Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1cec8d85-3d7e-4d93-930f-bfda6ed5959e","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:14.315Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59c73cbe-f719-471f-8ff0-e54cb9b44186","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:14.315Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59c73cbe-f719-471f-8ff0-e54cb9b44186","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:14.315Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59c73cbe-f719-471f-8ff0-e54cb9b44186","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:14.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59c73cbe-f719-471f-8ff0-e54cb9b44186"}
{"level":"info","ts":"2026-06-10T10:32:14.315Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"59c73cbe-f719-471f-8ff0-e54cb9b44186","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:17.316Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e94f95e2-b0ad-4655-9d2a-2c7144ce14fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:17.316Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e94f95e2-b0ad-4655-9d2a-2c7144ce14fc","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:17.316Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e94f95e2-b0ad-4655-9d2a-2c7144ce14fc","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:17.316Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e94f95e2-b0ad-4655-9d2a-2c7144ce14fc"}
{"level":"info","ts":"2026-06-10T10:32:17.316Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e94f95e2-b0ad-4655-9d2a-2c7144ce14fc","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:20.317Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"df7473dd-94c2-4dc0-9263-eb6af5f58a2f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:20.317Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"df7473dd-94c2-4dc0-9263-eb6af5f58a2f","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:20.317Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"df7473dd-94c2-4dc0-9263-eb6af5f58a2f","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:20.317Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"df7473dd-94c2-4dc0-9263-eb6af5f58a2f"}
{"level":"info","ts":"2026-06-10T10:32:20.317Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"df7473dd-94c2-4dc0-9263-eb6af5f58a2f","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:23.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9a8d7257-2618-439f-8aad-8c987437ef7f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:23.318Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9a8d7257-2618-439f-8aad-8c987437ef7f","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:23.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9a8d7257-2618-439f-8aad-8c987437ef7f","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:23.318Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9a8d7257-2618-439f-8aad-8c987437ef7f"}
{"level":"info","ts":"2026-06-10T10:32:23.318Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9a8d7257-2618-439f-8aad-8c987437ef7f","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:26.319Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8a0b85b8-10e9-428d-8b8c-a28e0e090b05","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:26.319Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8a0b85b8-10e9-428d-8b8c-a28e0e090b05","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:26.319Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8a0b85b8-10e9-428d-8b8c-a28e0e090b05","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:26.319Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8a0b85b8-10e9-428d-8b8c-a28e0e090b05"}
{"level":"info","ts":"2026-06-10T10:32:26.319Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8a0b85b8-10e9-428d-8b8c-a28e0e090b05","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:29.320Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"af35311a-c673-4bb1-b56a-7571ad73b084","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:29.320Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"af35311a-c673-4bb1-b56a-7571ad73b084","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:29.320Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"af35311a-c673-4bb1-b56a-7571ad73b084","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:29.320Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"af35311a-c673-4bb1-b56a-7571ad73b084"}
{"level":"info","ts":"2026-06-10T10:32:29.320Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"af35311a-c673-4bb1-b56a-7571ad73b084","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:32.321Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"a1057ced-6fc7-4899-beb6-13aa5775a139","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:32.321Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"a1057ced-6fc7-4899-beb6-13aa5775a139","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:32.321Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"a1057ced-6fc7-4899-beb6-13aa5775a139","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:32.321Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"a1057ced-6fc7-4899-beb6-13aa5775a139"}
{"level":"info","ts":"2026-06-10T10:32:32.321Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"a1057ced-6fc7-4899-beb6-13aa5775a139","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:35.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"576b916f-5fff-4d61-8abb-bfbb51c31838","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:35.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"576b916f-5fff-4d61-8abb-bfbb51c31838","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:35.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"576b916f-5fff-4d61-8abb-bfbb51c31838","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:35.322Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"576b916f-5fff-4d61-8abb-bfbb51c31838"}
{"level":"info","ts":"2026-06-10T10:32:35.322Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"576b916f-5fff-4d61-8abb-bfbb51c31838","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:38.323Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0187e689-4b65-43db-baf3-8ec34cb26ca8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:38.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0187e689-4b65-43db-baf3-8ec34cb26ca8","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:38.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0187e689-4b65-43db-baf3-8ec34cb26ca8","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:38.323Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0187e689-4b65-43db-baf3-8ec34cb26ca8"}
{"level":"info","ts":"2026-06-10T10:32:38.323Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0187e689-4b65-43db-baf3-8ec34cb26ca8","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:41.324Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0550122c-bd1b-49a6-9748-0f71338d8a33","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:41.324Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0550122c-bd1b-49a6-9748-0f71338d8a33","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:41.324Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0550122c-bd1b-49a6-9748-0f71338d8a33","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:41.324Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0550122c-bd1b-49a6-9748-0f71338d8a33"}
{"level":"info","ts":"2026-06-10T10:32:41.324Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0550122c-bd1b-49a6-9748-0f71338d8a33","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54"}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54"}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","podName":"stop-4bp9q-head-wdfdc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:32:41.508Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"89fdf07e-c18c-4be9-b0dc-54d791f97d54","seconds":300}
{"level":"info","ts":"2026-06-10T10:32:42.510Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb"}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb"}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","podName":"stop-4bp9q-head-wdfdc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","podName":"stop-4bp9q-head-wdfdc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:32:42.511Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"0a648e69-b5b2-42f5-8f6f-0159fcea1dfb","name":"stop-4bp9q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:32:42.523Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5b22ac2d-bf51-4364-aafa-2e154cd9ecc7","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:42.523Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7f50126f-85fb-4737-8cb0-db705bf4db47","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:42.523Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7f50126f-85fb-4737-8cb0-db705bf4db47","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:42.523Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7f50126f-85fb-4737-8cb0-db705bf4db47","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:42.523Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e380f9ec-68f3-4447-8f01-8f0ff69328c9","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:42.523Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5b22ac2d-bf51-4364-aafa-2e154cd9ecc7","pod":"stop-4bp9q-head-wdfdc","ip":"10.134.0.18"}
{"level":"info","ts":"2026-06-10T10:32:42.523Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5b22ac2d-bf51-4364-aafa-2e154cd9ecc7","pod":"stop-4bp9q-small-group-worker-dp9vm","ip":"10.133.0.14"}
{"level":"info","ts":"2026-06-10T10:32:42.523Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7f50126f-85fb-4737-8cb0-db705bf4db47"}
{"level":"info","ts":"2026-06-10T10:32:42.523Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7f50126f-85fb-4737-8cb0-db705bf4db47","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:42.524Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5b22ac2d-bf51-4364-aafa-2e154cd9ecc7","rayCluster":"stop-4bp9q","podIPs":["10.134.0.18","10.133.0.14"]}
{"level":"info","ts":"2026-06-10T10:32:42.604Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e380f9ec-68f3-4447-8f01-8f0ff69328c9","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:32:42.614Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e380f9ec-68f3-4447-8f01-8f0ff69328c9","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:32:42.614Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"e380f9ec-68f3-4447-8f01-8f0ff69328c9","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:44.324Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08284628-c0d9-4167-864b-0063ccf8e8e2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:44.324Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08284628-c0d9-4167-864b-0063ccf8e8e2","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:44.324Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08284628-c0d9-4167-864b-0063ccf8e8e2","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:44.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08284628-c0d9-4167-864b-0063ccf8e8e2"}
{"level":"info","ts":"2026-06-10T10:32:44.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"08284628-c0d9-4167-864b-0063ccf8e8e2","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2"}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2"}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","podName":"stop-4bp9q-small-group-worker-dp9vm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","podName":"stop-4bp9q-small-group-worker-dp9vm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:32:44.523Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7991ea32-2429-4657-9038-0f8a1b39d7c2","seconds":300}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3"}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3"}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:45.872Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","podName":"stop-4bp9q-head-wdfdc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:32:45.873Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","podName":"stop-4bp9q-head-wdfdc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:32:45.873Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:32:45.873Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"fc149bd1-0463-443d-bf0b-0ae7e016c9b3","seconds":300}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b"}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b"}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:46.880Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:46.881Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","podName":"stop-4bp9q-head-wdfdc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:32:46.881Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","podName":"stop-4bp9q-head-wdfdc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:32:46.881Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:32:46.881Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"f381acd6-5dde-430f-995b-126f3ca9464b","name":"stop-4bp9q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"345bd8db-55e4-4690-b0ef-f8fbeacb6785","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a624111f-3309-4ec9-9b7f-04b6487fcea2","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"345bd8db-55e4-4690-b0ef-f8fbeacb6785","pod":"stop-4bp9q-head-wdfdc","ip":"10.134.0.18"}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"345bd8db-55e4-4690-b0ef-f8fbeacb6785","pod":"stop-4bp9q-small-group-worker-dp9vm","ip":"10.133.0.14"}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"c40b05a1-383d-4670-88ba-52c994d23ff7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"c40b05a1-383d-4670-88ba-52c994d23ff7","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"c40b05a1-383d-4670-88ba-52c994d23ff7","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"345bd8db-55e4-4690-b0ef-f8fbeacb6785","rayCluster":"stop-4bp9q","podIPs":["10.134.0.18","10.133.0.14"]}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"c40b05a1-383d-4670-88ba-52c994d23ff7"}
{"level":"info","ts":"2026-06-10T10:32:46.893Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"c40b05a1-383d-4670-88ba-52c994d23ff7","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:46.904Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a624111f-3309-4ec9-9b7f-04b6487fcea2","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:32:46.909Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a291b54d-9eb3-41e3-856c-ea19a2b27af8","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:32:46.909Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a291b54d-9eb3-41e3-856c-ea19a2b27af8","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:32:46.909Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a291b54d-9eb3-41e3-856c-ea19a2b27af8","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:32:46.909Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a291b54d-9eb3-41e3-856c-ea19a2b27af8","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:32:46.914Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a624111f-3309-4ec9-9b7f-04b6487fcea2","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:32:46.914Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a624111f-3309-4ec9-9b7f-04b6487fcea2","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:47.325Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4f34a860-2b07-4c2b-985a-7a654c414324","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:47.325Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4f34a860-2b07-4c2b-985a-7a654c414324","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:47.325Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4f34a860-2b07-4c2b-985a-7a654c414324","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:47.325Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4f34a860-2b07-4c2b-985a-7a654c414324"}
{"level":"info","ts":"2026-06-10T10:32:47.325Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4f34a860-2b07-4c2b-985a-7a654c414324","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284"}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284"}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","podName":"stop-4bp9q-head-wdfdc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","podName":"stop-4bp9q-head-wdfdc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:32:48.893Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"8f7a0c54-67dc-4f36-9945-1ecf17f21284","seconds":300}
{"level":"info","ts":"2026-06-10T10:32:49.745Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"987935a5-a280-4d4a-a33f-f4c9a58817ad","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:32:49.745Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"987935a5-a280-4d4a-a33f-f4c9a58817ad","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:32:49.745Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"987935a5-a280-4d4a-a33f-f4c9a58817ad","pod":"fail-hxn94-small-group-worker-s48tx","ip":"10.132.0.42"}
{"level":"info","ts":"2026-06-10T10:32:49.745Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"987935a5-a280-4d4a-a33f-f4c9a58817ad","rayCluster":"fail-hxn94","podIPs":["10.134.0.17","10.132.0.42"]}
{"level":"info","ts":"2026-06-10T10:32:50.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9b771e1b-def1-4a29-b0c1-59b968fb6a10","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:50.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9b771e1b-def1-4a29-b0c1-59b968fb6a10","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:50.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9b771e1b-def1-4a29-b0c1-59b968fb6a10","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:50.326Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9b771e1b-def1-4a29-b0c1-59b968fb6a10"}
{"level":"info","ts":"2026-06-10T10:32:50.326Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"9b771e1b-def1-4a29-b0c1-59b968fb6a10","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:53.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c2077fc-a380-49c6-b974-bb315ac06788","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:53.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c2077fc-a380-49c6-b974-bb315ac06788","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:53.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c2077fc-a380-49c6-b974-bb315ac06788","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:53.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c2077fc-a380-49c6-b974-bb315ac06788"}
{"level":"info","ts":"2026-06-10T10:32:53.327Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1c2077fc-a380-49c6-b974-bb315ac06788","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb"}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb"}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","podName":"stop-4bp9q-small-group-worker-dp9vm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:32:53.545Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","podName":"stop-4bp9q-small-group-worker-dp9vm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:32:53.546Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:32:53.546Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"640f18d9-9777-4910-9392-3079954765bb","name":"stop-4bp9q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:32:53.558Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"18a2bce4-620b-4545-b3a4-f29aae48f9b9","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:53.558Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"aed7e50b-3f70-4e22-972b-8cfbd796bda1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:53.558Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"aed7e50b-3f70-4e22-972b-8cfbd796bda1","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:53.558Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"aed7e50b-3f70-4e22-972b-8cfbd796bda1","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:53.558Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"f0560075-8c7b-49fd-a668-5da878eff3e0","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:53.559Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"aed7e50b-3f70-4e22-972b-8cfbd796bda1"}
{"level":"info","ts":"2026-06-10T10:32:53.559Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"aed7e50b-3f70-4e22-972b-8cfbd796bda1","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:53.559Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"f0560075-8c7b-49fd-a668-5da878eff3e0","pod":"stop-4bp9q-head-wdfdc","ip":"10.134.0.18"}
{"level":"info","ts":"2026-06-10T10:32:53.559Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"f0560075-8c7b-49fd-a668-5da878eff3e0","pod":"stop-4bp9q-small-group-worker-dp9vm","ip":"10.133.0.14"}
{"level":"info","ts":"2026-06-10T10:32:53.559Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"f0560075-8c7b-49fd-a668-5da878eff3e0","rayCluster":"stop-4bp9q","podIPs":["10.134.0.18","10.133.0.14"]}
{"level":"info","ts":"2026-06-10T10:32:53.577Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"18a2bce4-620b-4545-b3a4-f29aae48f9b9","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:32:53.585Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"18a2bce4-620b-4545-b3a4-f29aae48f9b9","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:32:53.585Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"18a2bce4-620b-4545-b3a4-f29aae48f9b9","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:53.612Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"eed712d2-b156-4d30-8588-77da1a89d1de","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:53.612Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"eed712d2-b156-4d30-8588-77da1a89d1de","pod":"stop-4bp9q-head-wdfdc","ip":"10.134.0.18"}
{"level":"info","ts":"2026-06-10T10:32:53.612Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"eed712d2-b156-4d30-8588-77da1a89d1de","pod":"stop-4bp9q-small-group-worker-dp9vm","ip":"10.133.0.14"}
{"level":"info","ts":"2026-06-10T10:32:53.612Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"eed712d2-b156-4d30-8588-77da1a89d1de","rayCluster":"stop-4bp9q","podIPs":["10.134.0.18","10.133.0.14"]}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77"}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77"}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","podName":"stop-4bp9q-small-group-worker-dp9vm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","podName":"stop-4bp9q-small-group-worker-dp9vm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:31:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:32:55.559Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"dbc6b46f-67fd-4a81-9faf-b055489b0f77","seconds":300}
{"level":"info","ts":"2026-06-10T10:32:56.327Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7548f76e-0cce-4283-b698-0e4bcfd02459","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:56.327Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7548f76e-0cce-4283-b698-0e4bcfd02459","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:56.327Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7548f76e-0cce-4283-b698-0e4bcfd02459","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:56.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7548f76e-0cce-4283-b698-0e4bcfd02459"}
{"level":"info","ts":"2026-06-10T10:32:56.327Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7548f76e-0cce-4283-b698-0e4bcfd02459","RayCluster":"stop-4bp9q","State":""}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc"}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc"}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:57.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:57.902Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:32:57.902Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"3f6f29cf-273b-4782-a188-7a8d8e4f39fc","name":"stop-4bp9q","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:32:57.913Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"f50c561e-157b-4d7e-80c6-9445ba545514","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:57.913Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"d81ec3d0-c6e8-431d-9654-23b9050126ac","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:57.913Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0551508b-57aa-4306-a995-523660fee41b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:57.913Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0551508b-57aa-4306-a995-523660fee41b","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:57.913Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0551508b-57aa-4306-a995-523660fee41b","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:57.913Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"f50c561e-157b-4d7e-80c6-9445ba545514","pod":"stop-4bp9q-head-wdfdc","ip":"10.134.0.18"}
{"level":"info","ts":"2026-06-10T10:32:57.913Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"f50c561e-157b-4d7e-80c6-9445ba545514","pod":"stop-4bp9q-small-group-worker-dp9vm","ip":"10.133.0.14"}
{"level":"info","ts":"2026-06-10T10:32:57.913Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"f50c561e-157b-4d7e-80c6-9445ba545514","rayCluster":"stop-4bp9q","podIPs":["10.134.0.18","10.133.0.14"]}
{"level":"info","ts":"2026-06-10T10:32:57.914Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0551508b-57aa-4306-a995-523660fee41b"}
{"level":"info","ts":"2026-06-10T10:32:57.914Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0551508b-57aa-4306-a995-523660fee41b","head service name":"stop-4bp9q-head-svc","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:32:57.914Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0551508b-57aa-4306-a995-523660fee41b","head service URL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:32:57.914Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0551508b-57aa-4306-a995-523660fee41b","SubmissionMode":"HTTPMode","RayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:57.914Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0551508b-57aa-4306-a995-523660fee41b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:32:57.914Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0551508b-57aa-4306-a995-523660fee41b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:32:57.923Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f405f6a3-7ae5-494e-86f6-0bda3eaa2e2e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:57.923Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f405f6a3-7ae5-494e-86f6-0bda3eaa2e2e","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:57.923Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f405f6a3-7ae5-494e-86f6-0bda3eaa2e2e","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:57.923Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f405f6a3-7ae5-494e-86f6-0bda3eaa2e2e"}
{"level":"info","ts":"2026-06-10T10:32:57.925Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"d81ec3d0-c6e8-431d-9654-23b9050126ac","name":"stop-4bp9q-head"}
{"level":"info","ts":"2026-06-10T10:32:57.933Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f405f6a3-7ae5-494e-86f6-0bda3eaa2e2e","JobId":"stop-z77pv"}
{"level":"info","ts":"2026-06-10T10:32:57.933Z","logger":"controllers.RayJob","msg":"Submit a ray job","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f405f6a3-7ae5-494e-86f6-0bda3eaa2e2e","rayJob":"stop","jobInfo":"{\"entrypoint\":\"python /home/ray/jobs/stop.py\",\"submission_id\":\"stop-z77pv\"}"}
{"level":"info","ts":"2026-06-10T10:32:57.936Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"d81ec3d0-c6e8-431d-9654-23b9050126ac","name":"stop-4bp9q-workers"}
{"level":"info","ts":"2026-06-10T10:32:57.936Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"d81ec3d0-c6e8-431d-9654-23b9050126ac","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:59.329Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0e66e157-4efa-4f63-a37e-275ab4c5b8f2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:59.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0e66e157-4efa-4f63-a37e-275ab4c5b8f2","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:59.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0e66e157-4efa-4f63-a37e-275ab4c5b8f2","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:59.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0e66e157-4efa-4f63-a37e-275ab4c5b8f2"}
{"level":"info","ts":"2026-06-10T10:32:59.332Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0e66e157-4efa-4f63-a37e-275ab4c5b8f2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:32:59.332Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"0e66e157-4efa-4f63-a37e-275ab4c5b8f2","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:32:59.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"403f2f76-d9a0-4607-a10f-7542882e9b4f","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:32:59.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"403f2f76-d9a0-4607-a10f-7542882e9b4f","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:32:59.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"403f2f76-d9a0-4607-a10f-7542882e9b4f","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:32:59.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"403f2f76-d9a0-4607-a10f-7542882e9b4f"}
{"level":"info","ts":"2026-06-10T10:32:59.348Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"403f2f76-d9a0-4607-a10f-7542882e9b4f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8"}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8"}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","1 head service found":"stop-4bp9q-head-svc"}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","Found 1 head Pod":"stop-4bp9q-head-wdfdc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","head Pod":"stop-4bp9q-head-wdfdc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-4bp9q-head-wdfdc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","worker Pod":"stop-4bp9q-small-group-worker-dp9vm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-4bp9q-small-group-worker-dp9vm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:32:59.914Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"a48bb9bd-f405-4399-abad-4a316e3252d8","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:02.344Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e1962ef9-af35-46d5-8ad8-d15e5a249a47","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:02.344Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e1962ef9-af35-46d5-8ad8-d15e5a249a47","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:02.344Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e1962ef9-af35-46d5-8ad8-d15e5a249a47","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:02.344Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e1962ef9-af35-46d5-8ad8-d15e5a249a47"}
{"level":"info","ts":"2026-06-10T10:33:02.348Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e1962ef9-af35-46d5-8ad8-d15e5a249a47","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:33:02.348Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"e1962ef9-af35-46d5-8ad8-d15e5a249a47","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:33:02.364Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4447c031-a9db-4562-a5b5-1b7547a211a8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:02.364Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4447c031-a9db-4562-a5b5-1b7547a211a8","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:02.364Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4447c031-a9db-4562-a5b5-1b7547a211a8","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:02.364Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4447c031-a9db-4562-a5b5-1b7547a211a8"}
{"level":"info","ts":"2026-06-10T10:33:02.366Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"4447c031-a9db-4562-a5b5-1b7547a211a8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:33:05.365Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"cb51aee4-4d94-4a2e-a8ee-6ed4254a942f","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:05.365Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"cb51aee4-4d94-4a2e-a8ee-6ed4254a942f","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:05.365Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"cb51aee4-4d94-4a2e-a8ee-6ed4254a942f","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:05.365Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"cb51aee4-4d94-4a2e-a8ee-6ed4254a942f"}
{"level":"info","ts":"2026-06-10T10:33:05.368Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"cb51aee4-4d94-4a2e-a8ee-6ed4254a942f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:33:08.368Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"949f433a-abb5-4c8b-b625-5d7b3475a58b","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:08.368Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"949f433a-abb5-4c8b-b625-5d7b3475a58b","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:08.368Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"949f433a-abb5-4c8b-b625-5d7b3475a58b","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:08.368Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"949f433a-abb5-4c8b-b625-5d7b3475a58b"}
{"level":"info","ts":"2026-06-10T10:33:08.372Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"949f433a-abb5-4c8b-b625-5d7b3475a58b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:33:11.373Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1b0bfb72-fafa-4d8e-944c-bb79e6444dd3","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:11.373Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1b0bfb72-fafa-4d8e-944c-bb79e6444dd3","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:11.373Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1b0bfb72-fafa-4d8e-944c-bb79e6444dd3","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:11.373Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1b0bfb72-fafa-4d8e-944c-bb79e6444dd3"}
{"level":"info","ts":"2026-06-10T10:33:11.376Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"1b0bfb72-fafa-4d8e-944c-bb79e6444dd3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:33:14.376Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"2bb808e4-c3c3-4952-8512-422145deea05","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:14.376Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"2bb808e4-c3c3-4952-8512-422145deea05","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:14.376Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"2bb808e4-c3c3-4952-8512-422145deea05","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:14.376Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"2bb808e4-c3c3-4952-8512-422145deea05"}
{"level":"info","ts":"2026-06-10T10:33:14.379Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"2bb808e4-c3c3-4952-8512-422145deea05","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:33:17.380Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"156b59dd-147f-4bab-bcef-c039f5065895","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:17.380Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"156b59dd-147f-4bab-bcef-c039f5065895","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:17.380Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"156b59dd-147f-4bab-bcef-c039f5065895","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:17.380Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"156b59dd-147f-4bab-bcef-c039f5065895"}
{"level":"info","ts":"2026-06-10T10:33:17.384Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"156b59dd-147f-4bab-bcef-c039f5065895","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:33:20.385Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"74f7eedc-b5c0-48b2-809c-ffd6817f9e0e","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:20.385Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"74f7eedc-b5c0-48b2-809c-ffd6817f9e0e","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:20.385Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"74f7eedc-b5c0-48b2-809c-ffd6817f9e0e","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:20.385Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"74f7eedc-b5c0-48b2-809c-ffd6817f9e0e"}
{"level":"info","ts":"2026-06-10T10:33:20.388Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"74f7eedc-b5c0-48b2-809c-ffd6817f9e0e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:33:23.389Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7d598e68-dfad-4966-8a14-4566afeea216","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:23.389Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7d598e68-dfad-4966-8a14-4566afeea216","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:23.389Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7d598e68-dfad-4966-8a14-4566afeea216","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:23.390Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7d598e68-dfad-4966-8a14-4566afeea216"}
{"level":"info","ts":"2026-06-10T10:33:23.393Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7d598e68-dfad-4966-8a14-4566afeea216","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:32:58Z","endTime":"2026-06-10T10:33:20Z"},"jobId":"stop-z77pv","rayClusterName":"stop-4bp9q","dashboardURL":"stop-4bp9q-head-svc.test-ns-bbzgk.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:31:23Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:32:57Z","stateTransitionTimes":{"ready":"2026-06-10T10:32:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.18","serviceIP":"10.134.0.18","podName":"stop-4bp9q-head-wdfdc","serviceName":"stop-4bp9q-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:32:53Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:32:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:31:23Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:31:23Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:33:23.393Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"7d598e68-dfad-4966-8a14-4566afeea216","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-10T10:33:23.407Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8802b6ad-19dd-47c9-ad05-1c4f72f3b472","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:23.407Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"8802b6ad-19dd-47c9-ad05-1c4f72f3b472","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:33:23 +0000 UTC","Now":"2026-06-10T10:33:23.407Z","ShutdownTime":"2026-06-10T10:33:23.000Z"}
{"level":"info","ts":"2026-06-10T10:33:23.980Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"45283856-7ed6-4d9c-80fa-6f121b159ada","DeletionTimestamp":"2026-06-10 10:33:23 +0000 UTC"}
{"level":"info","ts":"2026-06-10T10:33:23.980Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"45283856-7ed6-4d9c-80fa-6f121b159ada","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:33:23.993Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"3989428f-3ca2-49a0-89d3-9e7806fd9a60"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"94eef2e8-1450-482e-9208-0d2a574bfbed","rayCluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"f5900bb4-a8a5-44a3-b553-230a82036b61"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5d704fb6-affe-430c-a9d8-1fb9b5d9bd30","namespacedName":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"reconcileID":"7dc4a6d5-8536-48a4-8565-6b298568ee09"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5d704fb6-affe-430c-a9d8-1fb9b5d9bd30","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5d704fb6-affe-430c-a9d8-1fb9b5d9bd30","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"b41b911f-90b9-4fec-80b2-12674667716a"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"94eef2e8-1450-482e-9208-0d2a574bfbed","pod":"stop-4bp9q-head-wdfdc","ip":"10.134.0.18"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"94eef2e8-1450-482e-9208-0d2a574bfbed","pod":"stop-4bp9q-small-group-worker-dp9vm","ip":"10.133.0.14"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"94eef2e8-1450-482e-9208-0d2a574bfbed","rayCluster":"stop-4bp9q","podIPs":["10.134.0.18","10.133.0.14"]}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5d704fb6-affe-430c-a9d8-1fb9b5d9bd30","namespace":"test-ns-bbzgk","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-06-10T10:33:24.014Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5d704fb6-affe-430c-a9d8-1fb9b5d9bd30","name":"kuberay-gateway-access","namespace":"test-ns-bbzgk","remainingClusters":1}
{"level":"info","ts":"2026-06-10T10:33:24.022Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5d704fb6-affe-430c-a9d8-1fb9b5d9bd30","name":"test-ns-bbzgk-stop-4bp9q","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:33:24.027Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5d704fb6-affe-430c-a9d8-1fb9b5d9bd30","configMap":"kube-rbac-proxy-config-stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:24.032Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5d704fb6-affe-430c-a9d8-1fb9b5d9bd30","serviceAccount":"stop-4bp9q-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:24.053Z","logger":"raycluster-resource","msg":"default","name":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:24.053Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-4bp9q","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"5d704fb6-affe-430c-a9d8-1fb9b5d9bd30","cluster":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a3a5d3e8-423e-4b73-8f7a-0d9a1e9245bf","namespacedName":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a3a5d3e8-423e-4b73-8f7a-0d9a1e9245bf"}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"86abfbec-aca8-47c1-8afd-10beec45eacf","namespace":"test-ns-bbzgk","clusterName":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a3a5d3e8-423e-4b73-8f7a-0d9a1e9245bf","namespace":"test-ns-bbzgk","authClusters":1}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a3a5d3e8-423e-4b73-8f7a-0d9a1e9245bf","namespace":"test-ns-bbzgk","authClusters":1}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"3226d89a-5050-4e66-9690-509353eaee67","namespacedName":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"3226d89a-5050-4e66-9690-509353eaee67"}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"bf1c2e12-6869-4083-b3c9-6fc7212b22f9"}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"3226d89a-5050-4e66-9690-509353eaee67","namespace":"test-ns-bbzgk","authClusters":1}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"3226d89a-5050-4e66-9690-509353eaee67","namespace":"test-ns-bbzgk","authClusters":1}
{"level":"info","ts":"2026-06-10T10:33:24.064Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"a1587c96-4037-4cee-8140-1dfcc78f0890"}
{"level":"info","ts":"2026-06-10T10:33:24.099Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"cbc2c445-e385-4ce6-a2e8-da7d4d2d7b01"}
{"level":"info","ts":"2026-06-10T10:33:24.099Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"65de5b8d-00fb-481c-8565-7a34b740af53","namespacedName":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:24.099Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"65de5b8d-00fb-481c-8565-7a34b740af53"}
{"level":"info","ts":"2026-06-10T10:33:24.099Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"bc09f115-efff-4596-8d33-121018089bd7"}
{"level":"error","ts":"2026-06-10T10:33:24.099Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"86abfbec-aca8-47c1-8afd-10beec45eacf","issuer":"ray-selfsigned-issuer-stop-4bp9q","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-stop-4bp9q\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.099Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"65de5b8d-00fb-481c-8565-7a34b740af53","namespace":"test-ns-bbzgk","authClusters":1}
{"level":"info","ts":"2026-06-10T10:33:24.100Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"65de5b8d-00fb-481c-8565-7a34b740af53","namespace":"test-ns-bbzgk","authClusters":1}
{"level":"error","ts":"2026-06-10T10:33:24.103Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"86abfbec-aca8-47c1-8afd-10beec45eacf","certificate":"ray-head-cert-stop-4bp9q","error":"certificates.cert-manager.io \"ray-head-cert-stop-4bp9q\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.131Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9f1c2947-7c3c-439c-a48e-3487064ab496","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:24.131Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9f1c2947-7c3c-439c-a48e-3487064ab496","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:33:24.138Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"86abfbec-aca8-47c1-8afd-10beec45eacf","errorCount":2}
{"level":"info","ts":"2026-06-10T10:33:24.138Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"86abfbec-aca8-47c1-8afd-10beec45eacf"}
{"level":"error","ts":"2026-06-10T10:33:24.138Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"86abfbec-aca8-47c1-8afd-10beec45eacf","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-stop-4bp9q\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.144Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ddc42a4c-4f7a-425f-aaf7-d785e0c49288","namespace":"test-ns-bbzgk","clusterName":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:24.144Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-4bp9q","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"stop-4bp9q","reconcileID":"ddc42a4c-4f7a-425f-aaf7-d785e0c49288","namespace":"test-ns-bbzgk","clusterName":"stop-4bp9q"}
{"level":"info","ts":"2026-06-10T10:33:24.199Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9f1c2947-7c3c-439c-a48e-3487064ab496"}
{"level":"info","ts":"2026-06-10T10:33:24.199Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9f1c2947-7c3c-439c-a48e-3487064ab496","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:33:24.199Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9f1c2947-7c3c-439c-a48e-3487064ab496","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfldr","rayClusterName":"long-running-gcmch","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:33:24.199Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9f1c2947-7c3c-439c-a48e-3487064ab496","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:33:24.210Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"47c0baaf-fd47-481c-ba20-a2268955b98b","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:24.210Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"47c0baaf-fd47-481c-ba20-a2268955b98b"}
{"level":"info","ts":"2026-06-10T10:33:24.210Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"47c0baaf-fd47-481c-ba20-a2268955b98b","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:33:24.210Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"47c0baaf-fd47-481c-ba20-a2268955b98b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-bpggd","rayClusterName":"long-running-8nzpt","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:33:24.210Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"47c0baaf-fd47-481c-ba20-a2268955b98b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:33:24.216Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"47c0baaf-fd47-481c-ba20-a2268955b98b","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-10T10:33:24.216Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"47c0baaf-fd47-481c-ba20-a2268955b98b"}
{"level":"error","ts":"2026-06-10T10:33:24.216Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"47c0baaf-fd47-481c-ba20-a2268955b98b","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.216Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"397c5bfc-c9df-4efa-b421-407d62033fe8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:24.216Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"397c5bfc-c9df-4efa-b421-407d62033fe8","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.216Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"397c5bfc-c9df-4efa-b421-407d62033fe8","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.216Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"397c5bfc-c9df-4efa-b421-407d62033fe8","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.221Z","logger":"raycluster-resource","msg":"default","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.221Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-gcmch","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:24.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"397c5bfc-c9df-4efa-b421-407d62033fe8","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.238Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97"}
{"level":"info","ts":"2026-06-10T10:33:24.238Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97"}
{"level":"info","ts":"2026-06-10T10:33:24.238Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","namespacedName":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.238Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b980d020-461a-426a-ba8a-883a3638e14e","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.238Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","mode":"IntegratedOAuth","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.238Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.238Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.238Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.238Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.243Z","logger":"raycluster-resource","msg":"default","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.243Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-gcmch","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:24.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"397c5bfc-c9df-4efa-b421-407d62033fe8"}
{"level":"info","ts":"2026-06-10T10:33:24.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"397c5bfc-c9df-4efa-b421-407d62033fe8","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:24.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"fcc84c9d-b83f-48d2-8c39-55daf16ed658","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:24.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"fcc84c9d-b83f-48d2-8c39-55daf16ed658","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"fcc84c9d-b83f-48d2-8c39-55daf16ed658","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"fcc84c9d-b83f-48d2-8c39-55daf16ed658"}
{"level":"info","ts":"2026-06-10T10:33:24.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"fcc84c9d-b83f-48d2-8c39-55daf16ed658","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:24.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7a2f604e-237c-40ec-8029-222a5072f8ca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:24.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7a2f604e-237c-40ec-8029-222a5072f8ca","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7a2f604e-237c-40ec-8029-222a5072f8ca","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.300Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.300Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7a2f604e-237c-40ec-8029-222a5072f8ca"}
{"level":"info","ts":"2026-06-10T10:33:24.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7a2f604e-237c-40ec-8029-222a5072f8ca","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:24.307Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","name":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:24.307Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","error":"head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"error","ts":"2026-06-10T10:33:24.307Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.307Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"info","ts":"2026-06-10T10:33:24.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.307Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:33:24.307Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","name":"long-running-gcmch","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:24.308Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b980d020-461a-426a-ba8a-883a3638e14e","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.308Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"651aabba-9117-40bb-aa99-56f60aa8af2e","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.308Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","name":"long-running-gcmch-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:24.311Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538"}
{"level":"info","ts":"2026-06-10T10:33:24.312Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","rayCluster":"long-running-gcmch","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:33:24.312Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.317Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","name":"kuberay-gateway-access","namespace":"test-ns-nrv9s","operation":"created"}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","name":"long-running-gcmch","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"long-running-gcmch"}}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97"}
{"level":"error","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d6b42032-c94c-4047-ad23-099bdb4c8c97","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8"}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8"}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","error":"head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"error","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:33:24.318Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","name":"long-running-gcmch","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:24.399Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"651aabba-9117-40bb-aa99-56f60aa8af2e","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:24.399Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","name":"test-ns-nrv9s-long-running-gcmch","operation":"created"}
{"level":"info","ts":"2026-06-10T10:33:24.400Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.401Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8"}
{"level":"error","ts":"2026-06-10T10:33:24.401Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"9aa42987-d5a9-4247-8739-21b88bd5f9c8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.499Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c"}
{"level":"info","ts":"2026-06-10T10:33:24.499Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c"}
{"level":"info","ts":"2026-06-10T10:33:24.499Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:24.499Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","error":"head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"error","ts":"2026-06-10T10:33:24.499Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.499Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"info","ts":"2026-06-10T10:33:24.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.500Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:33:24.500Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","name":"long-running-gcmch","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:24.499Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538"}
{"level":"info","ts":"2026-06-10T10:33:24.502Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","rayCluster":"long-running-gcmch","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:33:24.505Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"651aabba-9117-40bb-aa99-56f60aa8af2e","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:24.505Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"651aabba-9117-40bb-aa99-56f60aa8af2e","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.505Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"2aa26adf-99e6-4b7d-b93b-7af9c491d891","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9334acd0-5d2d-44f6-bc43-c276b1d29c47","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:24.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9334acd0-5d2d-44f6-bc43-c276b1d29c47","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9334acd0-5d2d-44f6-bc43-c276b1d29c47","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.599Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9334acd0-5d2d-44f6-bc43-c276b1d29c47"}
{"level":"info","ts":"2026-06-10T10:33:24.599Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9334acd0-5d2d-44f6-bc43-c276b1d29c47","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:24.600Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","name":"long-running-gcmch","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"long-running-gcmch"}}
{"level":"info","ts":"2026-06-10T10:33:24.600Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c"}
{"level":"error","ts":"2026-06-10T10:33:24.600Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b9315159-aa88-441a-a4e6-49c2e07ef08c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.600Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538"}
{"level":"info","ts":"2026-06-10T10:33:24.600Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","rayCluster":"long-running-gcmch","podIPs":[]}
{"level":"error","ts":"2026-06-10T10:33:24.600Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.600Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538"}
{"level":"error","ts":"2026-06-10T10:33:24.600Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"474e6b65-cc05-4c6d-a4fd-308c67d81538","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-gcmch\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.600Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"a31f8251-166e-4f84-a2c8-a96285001e22","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.605Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"2aa26adf-99e6-4b7d-b93b-7af9c491d891","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:24.617Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"a31f8251-166e-4f84-a2c8-a96285001e22"}
{"level":"error","ts":"2026-06-10T10:33:24.617Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"a31f8251-166e-4f84-a2c8-a96285001e22","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-gcmch\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:33:24.617Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"a31f8251-166e-4f84-a2c8-a96285001e22","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-gcmch\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.617Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"a31f8251-166e-4f84-a2c8-a96285001e22"}
{"level":"error","ts":"2026-06-10T10:33:24.617Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"a31f8251-166e-4f84-a2c8-a96285001e22","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-gcmch\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.617Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b3fccf7d-8598-451a-85b1-320dc2f6b2b9","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.617Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b3fccf7d-8598-451a-85b1-320dc2f6b2b9"}
{"level":"info","ts":"2026-06-10T10:33:24.620Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b"}
{"level":"info","ts":"2026-06-10T10:33:24.620Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b"}
{"level":"info","ts":"2026-06-10T10:33:24.621Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:24.621Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b","error":"head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"error","ts":"2026-06-10T10:33:24.621Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.621Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"info","ts":"2026-06-10T10:33:24.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.621Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.621Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b"}
{"level":"error","ts":"2026-06-10T10:33:24.621Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"65f9d063-d714-4b31-b6c8-884004c3e34b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","name":"kube-rbac-proxy-config-long-running-gcmch","operation":"created"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33","error":"head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"error","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"f400d8a6-d737-48fd-9aeb-fe6078d9e008","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33"}
{"level":"error","ts":"2026-06-10T10:33:24.699Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"cd68f018-e429-43d1-a528-a1962a2ebf33","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"f400d8a6-d737-48fd-9aeb-fe6078d9e008"}
{"level":"info","ts":"2026-06-10T10:33:24.699Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"2aa26adf-99e6-4b7d-b93b-7af9c491d891","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:24.700Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"2aa26adf-99e6-4b7d-b93b-7af9c491d891","cluster":"long-running-gcmch"}
{"level":"error","ts":"2026-06-10T10:33:24.707Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc"}
{"level":"error","ts":"2026-06-10T10:33:24.799Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"9a518b13-84e5-4b92-9c13-728484e318fc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"d242d28d-a82f-46a3-afb7-aa84d05b0f09","namespacedName":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"d242d28d-a82f-46a3-afb7-aa84d05b0f09","mode":"IntegratedOAuth","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"d242d28d-a82f-46a3-afb7-aa84d05b0f09","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834"}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834"}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834","error":"head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"error","ts":"2026-06-10T10:33:24.799Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.800Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834"}
{"level":"error","ts":"2026-06-10T10:33:24.800Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2424b5c6-bce3-42f7-afbd-e1f6ce27c834","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.813Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"d242d28d-a82f-46a3-afb7-aa84d05b0f09","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.813Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"c1450961-d8ef-467c-8bd7-c4a31b794661","namespacedName":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.813Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"c1450961-d8ef-467c-8bd7-c4a31b794661","mode":"IntegratedOAuth","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.813Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"c1450961-d8ef-467c-8bd7-c4a31b794661","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.813Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4c5aad9a-cce4-4900-95bc-e9539c678cd1","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.813Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"09b79aa8-10e5-41c7-8c25-42ee6239774a","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.813Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c53084e7-9ed8-4df1-86fd-1288f7d8f78e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:24.814Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c53084e7-9ed8-4df1-86fd-1288f7d8f78e","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.814Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c53084e7-9ed8-4df1-86fd-1288f7d8f78e","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.813Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"09b79aa8-10e5-41c7-8c25-42ee6239774a"}
{"level":"info","ts":"2026-06-10T10:33:24.814Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c53084e7-9ed8-4df1-86fd-1288f7d8f78e"}
{"level":"info","ts":"2026-06-10T10:33:24.814Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c53084e7-9ed8-4df1-86fd-1288f7d8f78e","RayCluster":"long-running-gcmch","State":""}
{"level":"error","ts":"2026-06-10T10:33:24.821Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"c1450961-d8ef-467c-8bd7-c4a31b794661","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.821Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"c1450961-d8ef-467c-8bd7-c4a31b794661"}
{"level":"error","ts":"2026-06-10T10:33:24.821Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"c1450961-d8ef-467c-8bd7-c4a31b794661","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.827Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"505f7725-1e80-48cd-80c8-e4e7fbdb763e","namespacedName":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:24.827Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"505f7725-1e80-48cd-80c8-e4e7fbdb763e","mode":"IntegratedOAuth","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.827Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"505f7725-1e80-48cd-80c8-e4e7fbdb763e","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.899Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4c5aad9a-cce4-4900-95bc-e9539c678cd1","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:24.899Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"505f7725-1e80-48cd-80c8-e4e7fbdb763e","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4c5aad9a-cce4-4900-95bc-e9539c678cd1","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:24.921Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4c5aad9a-cce4-4900-95bc-e9539c678cd1","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50"}
{"level":"info","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50"}
{"level":"info","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50","error":"head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"error","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"info","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50"}
{"level":"error","ts":"2026-06-10T10:33:24.961Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"8d3474ee-954c-45ee-827b-d73895807c50","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5"}
{"level":"info","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5"}
{"level":"info","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5","error":"head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"error","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"info","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5"}
{"level":"error","ts":"2026-06-10T10:33:25.281Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"44ecd6e4-2ddc-4284-b03a-26cb5b0cf0b5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:25.922Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8"}
{"level":"info","ts":"2026-06-10T10:33:25.922Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8"}
{"level":"info","ts":"2026-06-10T10:33:25.922Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:25.922Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8","error":"head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"error","ts":"2026-06-10T10:33:25.922Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:25.923Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found"}
{"level":"info","ts":"2026-06-10T10:33:25.923Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:25.923Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:25.923Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-gcmch","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:25.923Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8"}
{"level":"error","ts":"2026-06-10T10:33:25.923Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"73c0a0e3-0ca3-419f-86a4-6d4cbbf9abc8","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-gcmch not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:26.407Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-bbzgk"},"reconcileID":"07276f05-44b0-4dd8-84ed-b61f71f48783"}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304"}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304"}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304"}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304"}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","mode":"IntegratedOAuth","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","cluster":"long-running-gcmch","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","cluster":"long-running-gcmch","authType":"IntegratedOAuth","serviceAccount":"long-running-gcmch-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-gcmch","ray.io/group":"headgroup","ray.io/identifier":"long-running-gcmch-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:33:27.204Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","enabled":true}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","name":"long-running-gcmch-head-qr4ms"}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304"}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","nodeType":"worker","rayStartParams":{"address":"long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","rayStartCmd":"ray start  --address=long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:27.224Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:27.225Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","enabled":true}
{"level":"info","ts":"2026-06-10T10:33:27.239Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","name":"long-running-gcmch-small-group-worker-tzxbm"}
{"level":"info","ts":"2026-06-10T10:33:27.240Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.240Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.240Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-gcmch-head-svc"},"newHead":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"}}
{"level":"info","ts":"2026-06-10T10:33:27.240Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"63de114d-c5f1-496f-8892-a3fe9744f304","name":"long-running-gcmch","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5602c6dc-6b46-4344-9a9a-8dc2da286cf2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5602c6dc-6b46-4344-9a9a-8dc2da286cf2","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"a3416a72-ac68-41e7-ac08-9266390655e8","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5602c6dc-6b46-4344-9a9a-8dc2da286cf2","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"a3416a72-ac68-41e7-ac08-9266390655e8","rayCluster":"long-running-gcmch","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5602c6dc-6b46-4344-9a9a-8dc2da286cf2"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4d0a2c99-6fb8-4ce2-a9e8-3e6166c3be02","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5602c6dc-6b46-4344-9a9a-8dc2da286cf2","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"3ca3e8e3-23ed-440a-a839-b7216515dd1b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:27.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"3ca3e8e3-23ed-440a-a839-b7216515dd1b","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"3ca3e8e3-23ed-440a-a839-b7216515dd1b","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:27.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"3ca3e8e3-23ed-440a-a839-b7216515dd1b"}
{"level":"info","ts":"2026-06-10T10:33:27.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"3ca3e8e3-23ed-440a-a839-b7216515dd1b","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:27.300Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-gcmch-head-svc"},"newHead":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"}}
{"level":"info","ts":"2026-06-10T10:33:27.300Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","name":"long-running-gcmch","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","name":"long-running-gcmch","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"long-running-gcmch"}}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19"}
{"level":"error","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ce5c7b47-6b76-449b-9215-1698ea379c19","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff"}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff"}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","podName":"long-running-gcmch-small-group-worker-tzxbm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","podName":"long-running-gcmch-small-group-worker-tzxbm","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:33:27.306Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"17c2aae3-4c60-4167-8435-522d685a34ff","name":"long-running-gcmch","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:27.309Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4d0a2c99-6fb8-4ce2-a9e8-3e6166c3be02","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354"}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354"}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.317Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.318Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:33:27.318Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","name":"long-running-gcmch","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:27.318Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"10b7eae6-f5bc-4e9e-b605-ad2888ddef46","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"042ed71f-8b2a-4983-83d3-605a1d3e45b7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:27.318Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"042ed71f-8b2a-4983-83d3-605a1d3e45b7","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"042ed71f-8b2a-4983-83d3-605a1d3e45b7","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:27.318Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"10b7eae6-f5bc-4e9e-b605-ad2888ddef46","rayCluster":"long-running-gcmch","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:33:27.318Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"042ed71f-8b2a-4983-83d3-605a1d3e45b7"}
{"level":"info","ts":"2026-06-10T10:33:27.318Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"042ed71f-8b2a-4983-83d3-605a1d3e45b7","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:27.320Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4d0a2c99-6fb8-4ce2-a9e8-3e6166c3be02","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:27.320Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4d0a2c99-6fb8-4ce2-a9e8-3e6166c3be02","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.320Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"70075f8b-140d-46c9-b68c-b145084f1feb","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.324Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","name":"long-running-gcmch","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"long-running-gcmch"}}
{"level":"info","ts":"2026-06-10T10:33:27.324Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354"}
{"level":"error","ts":"2026-06-10T10:33:27.324Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"ebbb5173-aaa5-4dbf-ab64-98e5ffe94354","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-gcmch\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73"}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73"}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:27.330Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"c4067a81-1319-45f0-88ec-66c598ec4c73","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:27.399Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"70075f8b-140d-46c9-b68c-b145084f1feb","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:27.408Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"70075f8b-140d-46c9-b68c-b145084f1feb","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:27.408Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"70075f8b-140d-46c9-b68c-b145084f1feb","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558"}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558"}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.676Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:27.676Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"997f9227-b43d-4937-a109-535575daa558","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:27.696Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe"}
{"level":"info","ts":"2026-06-10T10:33:27.696Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe"}
{"level":"info","ts":"2026-06-10T10:33:27.696Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:27.697Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a1827975-35da-491e-801a-ba3e19f84dfe","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087"}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087"}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:28.025Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1b525430-29c8-4c19-89f7-ef9454e3e087","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:28.688Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512"}
{"level":"info","ts":"2026-06-10T10:33:28.688Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512"}
{"level":"info","ts":"2026-06-10T10:33:28.688Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:28.688Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"newHead":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"}}
{"level":"info","ts":"2026-06-10T10:33:28.689Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"13920c0d-32d0-402c-8305-03b3801e2512","name":"long-running-gcmch","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:28.700Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fdb7ca57-92f4-4741-b5e9-5338e5d46682","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:28.700Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"d00c0e87-bb46-4cf8-bc0d-cb26abca9328","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:28.700Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fdb7ca57-92f4-4741-b5e9-5338e5d46682","pod":"long-running-gcmch-head-qr4ms","ip":"10.134.0.19"}
{"level":"info","ts":"2026-06-10T10:33:28.700Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fdb7ca57-92f4-4741-b5e9-5338e5d46682","pod":"long-running-gcmch-small-group-worker-tzxbm","ip":"10.133.0.15"}
{"level":"info","ts":"2026-06-10T10:33:28.700Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"8a7ba54d-e17c-488f-ae1a-a3b520c814f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:28.700Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"8a7ba54d-e17c-488f-ae1a-a3b520c814f9","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:28.700Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"8a7ba54d-e17c-488f-ae1a-a3b520c814f9","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:28.700Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"8a7ba54d-e17c-488f-ae1a-a3b520c814f9"}
{"level":"info","ts":"2026-06-10T10:33:28.700Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"8a7ba54d-e17c-488f-ae1a-a3b520c814f9","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:28.711Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"d00c0e87-bb46-4cf8-bc0d-cb26abca9328","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:28.712Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fdb7ca57-92f4-4741-b5e9-5338e5d46682","certificate":"ray-head-cert-long-running-gcmch","dnsNames":["long-running-gcmch-head-svc","localhost","long-running-gcmch-head-svc.test-ns-nrv9s.svc","long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.19","10.133.0.15","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:33:28.722Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"d00c0e87-bb46-4cf8-bc0d-cb26abca9328","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:28.722Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"d00c0e87-bb46-4cf8-bc0d-cb26abca9328","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:28.723Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fdb7ca57-92f4-4741-b5e9-5338e5d46682","certificate":"ray-worker-cert-long-running-gcmch","dnsNames":["long-running-gcmch-worker-svc","localhost","long-running-gcmch-small-group","long-running-gcmch-worker-svc.test-ns-nrv9s.svc","long-running-gcmch-worker-svc.test-ns-nrv9s.svc.cluster.local","*.long-running-gcmch-worker-svc.test-ns-nrv9s.svc","*.long-running-gcmch-worker-svc.test-ns-nrv9s.svc.cluster.local","*-worker-*.test-ns-nrv9s.svc","*-worker-*.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.19","10.133.0.15","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:33:28.723Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fdb7ca57-92f4-4741-b5e9-5338e5d46682","rayCluster":"long-running-gcmch","podIPs":["10.134.0.19","10.133.0.15"]}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f"}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f"}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:29.113Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"8e49a95e-12d6-4c82-bd61-1218d306ea5f","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b"}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b"}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:29.118Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.119Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:29.119Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"902cc14c-c1b6-4c83-b9a3-9c2fb77fd96b","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551"}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551"}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:29.136Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7051545-aeb0-4425-b616-444a0aaa8551","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420"}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420"}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-hxn94-small-group-worker-s48tx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:29.140Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"5b459d5c-cf47-464b-adf2-5e5de7548420","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:29.617Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754"}
{"level":"info","ts":"2026-06-10T10:33:29.617Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754"}
{"level":"info","ts":"2026-06-10T10:33:29.617Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.617Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","worker Pod":"fail-hxn94-small-group-worker-s48tx","shouldDelete":true,"reason":"The worker Pod fail-hxn94-small-group-worker-s48tx status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-10T10:33:29.626Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.626Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-06-10T10:33:29.626Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","oldReadyWorkerReplicas":1,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":1,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.626Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","name":"fail-hxn94","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:31:19Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:29Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:29.642Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754"}
{"level":"error","ts":"2026-06-10T10:33:29.643Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"38368065-c592-4fca-a1eb-a7a97608a754","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.643Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"77faaca5-5724-415e-8061-25ccba68d0f9","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:29.643Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a55a430e-ab08-4a43-b431-018c8e68af70","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:29.643Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"a8ad1040-ac2f-4984-b7a9-7e6306d74d85","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:29.643Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"a8ad1040-ac2f-4984-b7a9-7e6306d74d85","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:31:22 +0000 UTC","Now":"2026-06-10T10:33:29.643Z","ShutdownTime":"2026-06-10T10:31:22.000Z"}
{"level":"info","ts":"2026-06-10T10:33:29.643Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a55a430e-ab08-4a43-b431-018c8e68af70","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:33:29.643Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a55a430e-ab08-4a43-b431-018c8e68af70","pod":"fail-hxn94-small-group-worker-s48tx","ip":"10.132.0.42"}
{"level":"info","ts":"2026-06-10T10:33:29.643Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a55a430e-ab08-4a43-b431-018c8e68af70","rayCluster":"fail-hxn94","podIPs":["10.134.0.17","10.132.0.42"]}
{"level":"info","ts":"2026-06-10T10:33:29.648Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7adbf66-9143-4eed-89c5-09d0f30b1453"}
{"level":"info","ts":"2026-06-10T10:33:29.648Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7adbf66-9143-4eed-89c5-09d0f30b1453"}
{"level":"info","ts":"2026-06-10T10:33:29.648Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7adbf66-9143-4eed-89c5-09d0f30b1453","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.648Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"77faaca5-5724-415e-8061-25ccba68d0f9","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:29.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7adbf66-9143-4eed-89c5-09d0f30b1453","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7adbf66-9143-4eed-89c5-09d0f30b1453","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.648Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7adbf66-9143-4eed-89c5-09d0f30b1453","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-10T10:33:29.649Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7adbf66-9143-4eed-89c5-09d0f30b1453","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-10T10:33:29.649Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7adbf66-9143-4eed-89c5-09d0f30b1453","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:29.649Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c7adbf66-9143-4eed-89c5-09d0f30b1453","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:29.756Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b1eae63d-d943-46c0-b743-25db327d1bb6"}
{"level":"info","ts":"2026-06-10T10:33:29.756Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b1eae63d-d943-46c0-b743-25db327d1bb6"}
{"level":"info","ts":"2026-06-10T10:33:29.756Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b1eae63d-d943-46c0-b743-25db327d1bb6","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b1eae63d-d943-46c0-b743-25db327d1bb6","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b1eae63d-d943-46c0-b743-25db327d1bb6","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.756Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b1eae63d-d943-46c0-b743-25db327d1bb6","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-10T10:33:29.757Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b1eae63d-d943-46c0-b743-25db327d1bb6","podName":"fail-hxn94-small-group-worker-s48tx","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-10T10:33:29.757Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b1eae63d-d943-46c0-b743-25db327d1bb6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:29.757Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"b1eae63d-d943-46c0-b743-25db327d1bb6","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be"}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be"}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be"}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","nodeType":"worker","rayStartParams":{"address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","rayStartCmd":"ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.762Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:29.766Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.766Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:29.766Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:33:29Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}]}
{"level":"info","ts":"2026-06-10T10:33:29.766Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","name":"fail-hxn94","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:29Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:29Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:33:29Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:29.780Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be"}
{"level":"error","ts":"2026-06-10T10:33:29.780Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0b061283-fb13-45e5-8bcd-16137930d2be","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.780Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"4c935ac6-37c5-4cef-b995-f1253eeec689","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:29.780Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"719f460f-1395-4e45-8655-84ebaee2a304","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:29.780Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"4c935ac6-37c5-4cef-b995-f1253eeec689","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:33:29.780Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"b0591bcb-51ab-4a87-8b88-3359d44d9fab","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:29.780Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"b0591bcb-51ab-4a87-8b88-3359d44d9fab","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:31:22 +0000 UTC","Now":"2026-06-10T10:33:29.780Z","ShutdownTime":"2026-06-10T10:31:22.000Z"}
{"level":"info","ts":"2026-06-10T10:33:29.784Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"719f460f-1395-4e45-8655-84ebaee2a304","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:29.785Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0"}
{"level":"info","ts":"2026-06-10T10:33:29.785Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0"}
{"level":"info","ts":"2026-06-10T10:33:29.786Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.793Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"4c935ac6-37c5-4cef-b995-f1253eeec689","certificate":"ray-head-cert-fail-hxn94","dnsNames":["fail-hxn94-head-svc","localhost","fail-hxn94-head-svc.test-ns-bbzgk.svc","fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:33:29.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:29.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.799Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:29.799Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0"}
{"level":"info","ts":"2026-06-10T10:33:29.799Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","nodeType":"worker","rayStartParams":{"address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:29.799Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","rayStartCmd":"ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.799Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.799Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:29.802Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.802Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:29.802Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0"}
{"level":"error","ts":"2026-06-10T10:33:29.802Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c94de1de-e9e7-4da9-86bb-93c3551c3cd0","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.806Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"4c935ac6-37c5-4cef-b995-f1253eeec689","certificate":"ray-worker-cert-fail-hxn94","dnsNames":["fail-hxn94-worker-svc","localhost","fail-hxn94-small-group","fail-hxn94-worker-svc.test-ns-bbzgk.svc","fail-hxn94-worker-svc.test-ns-bbzgk.svc.cluster.local","*.fail-hxn94-worker-svc.test-ns-bbzgk.svc","*.fail-hxn94-worker-svc.test-ns-bbzgk.svc.cluster.local","*-worker-*.test-ns-bbzgk.svc","*-worker-*.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["10.134.0.17","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:33:29.806Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"4c935ac6-37c5-4cef-b995-f1253eeec689","rayCluster":"fail-hxn94","podIPs":["10.134.0.17"]}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea"}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea"}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea"}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","nodeType":"worker","rayStartParams":{"address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","rayStartCmd":"ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.813Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:29.816Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.816Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:29.817Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea"}
{"level":"error","ts":"2026-06-10T10:33:29.817Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c996061a-0329-4b13-959e-7e62a35729ea","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.837Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd"}
{"level":"info","ts":"2026-06-10T10:33:29.837Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd"}
{"level":"info","ts":"2026-06-10T10:33:29.837Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd"}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","nodeType":"worker","rayStartParams":{"address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","rayStartCmd":"ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.838Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:29.842Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.842Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:29.842Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd"}
{"level":"error","ts":"2026-06-10T10:33:29.842Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"92fbcbe0-9af7-45cc-bf50-db5843297bbd","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e"}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e"}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e"}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","nodeType":"worker","rayStartParams":{"address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","rayStartCmd":"ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.883Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:29.886Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.886Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:29.886Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e"}
{"level":"error","ts":"2026-06-10T10:33:29.887Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"edf04167-1560-40b2-86b2-40084d69344e","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18"}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18"}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-hxn94-head-nj5fr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18"}
{"level":"info","ts":"2026-06-10T10:33:29.967Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","nodeType":"worker","rayStartParams":{"address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:29.968Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","rayStartCmd":"ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.968Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:29.968Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:29.971Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:29.971Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:29.971Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18"}
{"level":"error","ts":"2026-06-10T10:33:29.971Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"4d150ad6-c51a-4b4a-af88-78ec4f411b18","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.077Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22"}
{"level":"info","ts":"2026-06-10T10:33:30.077Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22"}
{"level":"info","ts":"2026-06-10T10:33:30.077Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:30.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22","Found 1 head Pod":"fail-hxn94-head-nj5fr","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-10 10:30:52 +0000 UTC,FinishedAt:2026-06-10 10:33:29 +0000 UTC,ContainerID:cri-o://b94d06d91e17749e597178977a422e512e7ec50d4245ce1a18ccba3e753ffd82,}"}
{"level":"info","ts":"2026-06-10T10:33:30.077Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22","head Pod":"fail-hxn94-head-nj5fr","shouldDelete":true,"reason":"The head Pod fail-hxn94-head-nj5fr status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-10T10:33:30.081Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod fail-hxn94-head-nj5fr status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.081Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22","error":"The head Pod fail-hxn94-head-nj5fr status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-10T10:33:30.081Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22","old conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:33:29Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:30Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:33:29Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}]}
{"level":"info","ts":"2026-06-10T10:33:30.081Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22","name":"fail-hxn94","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:29Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:31:04Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:33:29Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:30Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:33:29Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:30.092Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22"}
{"level":"error","ts":"2026-06-10T10:33:30.092Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"0628d7ee-cab2-4fe1-862f-675985034a22","error":"The head Pod fail-hxn94-head-nj5fr status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.093Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1329b3cf-944b-4b30-92ea-abe5076ee28c","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:30.093Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1329b3cf-944b-4b30-92ea-abe5076ee28c","pod":"fail-hxn94-head-nj5fr","ip":"10.134.0.17"}
{"level":"info","ts":"2026-06-10T10:33:30.093Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"5203a0d6-c1b2-4a33-9ecb-b5101d24eff6","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:30.093Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1329b3cf-944b-4b30-92ea-abe5076ee28c","rayCluster":"fail-hxn94","podIPs":["10.134.0.17"]}
{"level":"info","ts":"2026-06-10T10:33:30.093Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"3b56f84a-46b8-4bdd-a2de-5be01657fbc5","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:30.093Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"3b56f84a-46b8-4bdd-a2de-5be01657fbc5","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:31:22 +0000 UTC","Now":"2026-06-10T10:33:30.093Z","ShutdownTime":"2026-06-10T10:31:22.000Z"}
{"level":"info","ts":"2026-06-10T10:33:30.097Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"5203a0d6-c1b2-4a33-9ecb-b5101d24eff6","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6"}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6"}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6"}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","nodeType":"worker","rayStartParams":{"address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","rayStartCmd":"ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:30.132Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:30.136Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.136Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:30.136Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6"}
{"level":"error","ts":"2026-06-10T10:33:30.136Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"3b506723-3ee3-4009-8316-f1b8c4ed95b6","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"790eed46-572f-4303-a19b-b34bd55c8e0b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:30.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"790eed46-572f-4303-a19b-b34bd55c8e0b","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:30.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"790eed46-572f-4303-a19b-b34bd55c8e0b","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:30.301Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"790eed46-572f-4303-a19b-b34bd55c8e0b"}
{"level":"info","ts":"2026-06-10T10:33:30.301Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"790eed46-572f-4303-a19b-b34bd55c8e0b","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662"}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662"}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662"}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","nodeType":"worker","rayStartParams":{"address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","rayStartCmd":"ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:30.700Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:30.703Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.703Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662"}
{"level":"error","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"75dec9cf-2da1-4eb5-900a-37078d36a662","error":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:30.704Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b622d20f-e20c-404e-8a59-c1dd7f3d74b6","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7"}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7"}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7"}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7"}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","mode":"IntegratedOAuth","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","cluster":"fail-hxn94","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","cluster":"fail-hxn94","authType":"IntegratedOAuth","serviceAccount":"fail-hxn94-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-hxn94","ray.io/group":"headgroup","ray.io/identifier":"fail-hxn94-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:33:30.706Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:30.709Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-hxn94-head-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.709Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","error":"FailedCreateHeadPod\npods \"fail-hxn94-head-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:30.709Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:30.709Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:30.709Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:30.709Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"newHead":{"serviceName":"fail-hxn94-head-svc"}}
{"level":"info","ts":"2026-06-10T10:33:30.709Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","name":"fail-hxn94","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.17","serviceIP":"10.134.0.17","podName":"fail-hxn94-head-nj5fr","serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:30Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:33:29Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"fail-hxn94-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:31:19Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-hxn94-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:31:19Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:29:19Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:29:19Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:33:29Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"fail-hxn94-head-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:30.722Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7"}
{"level":"error","ts":"2026-06-10T10:33:30.722Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"c2bb73fc-25f7-4672-964c-48cfc9fc29c7","error":"FailedCreateHeadPod\npods \"fail-hxn94-head-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.723Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"0c409b7f-50a7-4163-a9c0-82226e8f3774","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:30.723Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"28cdeaf7-33da-452b-8212-5cf2ab9f54f4","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:30.723Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"eea7f760-025d-417c-9ddf-85e205edf9a2","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:30.723Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"eea7f760-025d-417c-9ddf-85e205edf9a2","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:31:22 +0000 UTC","Now":"2026-06-10T10:33:30.723Z","ShutdownTime":"2026-06-10T10:31:22.000Z"}
{"level":"info","ts":"2026-06-10T10:33:30.727Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"28cdeaf7-33da-452b-8212-5cf2ab9f54f4","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:30.736Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"0c409b7f-50a7-4163-a9c0-82226e8f3774","certificate":"ray-head-cert-fail-hxn94","dnsNames":["fail-hxn94-head-svc","localhost","fail-hxn94-head-svc.test-ns-bbzgk.svc","fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:33:30.751Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"0c409b7f-50a7-4163-a9c0-82226e8f3774","certificate":"ray-worker-cert-fail-hxn94","dnsNames":["fail-hxn94-worker-svc","localhost","fail-hxn94-small-group","fail-hxn94-worker-svc.test-ns-bbzgk.svc","fail-hxn94-worker-svc.test-ns-bbzgk.svc.cluster.local","*.fail-hxn94-worker-svc.test-ns-bbzgk.svc","*.fail-hxn94-worker-svc.test-ns-bbzgk.svc.cluster.local","*-worker-*.test-ns-bbzgk.svc","*-worker-*.test-ns-bbzgk.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:33:30.751Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"0c409b7f-50a7-4163-a9c0-82226e8f3774"}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea"}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea"}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","1 head service found":"fail-hxn94-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea"}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea"}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","mode":"IntegratedOAuth","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","cluster":"fail-hxn94","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","cluster":"fail-hxn94","authType":"IntegratedOAuth","serviceAccount":"fail-hxn94-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-hxn94","ray.io/group":"headgroup","ray.io/identifier":"fail-hxn94-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-hxn94-head-svc.test-ns-bbzgk.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:33:30.777Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","enabled":true}
{"level":"error","ts":"2026-06-10T10:33:30.802Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"fail-hxn94-head-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:30.802Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","error":"FailedCreateHeadPod\npods \"fail-hxn94-head-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:33:30.802Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:30.802Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:30.802Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","filter labels":["test-ns-bbzgk",{"ray.io/cluster":"fail-hxn94","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:30.802Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea"}
{"level":"error","ts":"2026-06-10T10:33:30.802Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"7a5a7508-39f1-4d3b-9de2-b7b8890f17ea","error":"FailedCreateHeadPod\npods \"fail-hxn94-head-\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:33.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bb01c7eb-fc53-400e-afc6-fb2cd1032646","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:33.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bb01c7eb-fc53-400e-afc6-fb2cd1032646","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:33.302Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bb01c7eb-fc53-400e-afc6-fb2cd1032646","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:33.302Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bb01c7eb-fc53-400e-afc6-fb2cd1032646"}
{"level":"info","ts":"2026-06-10T10:33:33.302Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bb01c7eb-fc53-400e-afc6-fb2cd1032646","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31"}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31"}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","podName":"long-running-gcmch-head-qr4ms","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:33.710Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"b006b393-defc-442e-9cf4-6b73eb9f0f31","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:34.204Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cd1ff994-ccfa-4aed-87c6-f342e5fda49a","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.209Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cd1ff994-ccfa-4aed-87c6-f342e5fda49a","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.209Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"8b0af47f-4fec-49db-ab25-c9b6c356aeec","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.213Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"8b0af47f-4fec-49db-ab25-c9b6c356aeec","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd0911d7-2949-429c-8656-9da3298a33a5","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd0911d7-2949-429c-8656-9da3298a33a5","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"12415e07-464c-4e75-a5b4-b1660a2bc5d6","namespacedName":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"7b026e64-15b0-4eae-86b3-7c5f39ec1984"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"12415e07-464c-4e75-a5b4-b1660a2bc5d6","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"12415e07-464c-4e75-a5b4-b1660a2bc5d6","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"reconcileID":"6103886d-2e0e-42bf-9bc8-c494ecc4dcb7"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"12415e07-464c-4e75-a5b4-b1660a2bc5d6","namespace":"test-ns-bbzgk","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"12415e07-464c-4e75-a5b4-b1660a2bc5d6","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"e7d04687-62f9-47e3-9912-68558f3f22bd","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-06-10T10:33:34.395Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"e7d04687-62f9-47e3-9912-68558f3f22bd","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:31:22 +0000 UTC","Now":"2026-06-10T10:33:34.395Z","ShutdownTime":"2026-06-10T10:31:22.000Z"}
{"level":"error","ts":"2026-06-10T10:33:34.397Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd0911d7-2949-429c-8656-9da3298a33a5","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-hxn94\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:33:34.397Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd0911d7-2949-429c-8656-9da3298a33a5","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-hxn94\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.397Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd0911d7-2949-429c-8656-9da3298a33a5"}
{"level":"error","ts":"2026-06-10T10:33:34.397Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"dd0911d7-2949-429c-8656-9da3298a33a5","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-hxn94\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.399Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"12415e07-464c-4e75-a5b4-b1660a2bc5d6","name":"test-ns-bbzgk-fail-hxn94","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:33:34.403Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a14c6ee3-f966-40b6-9477-dc63f9c5d8ce","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.403Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a14c6ee3-f966-40b6-9477-dc63f9c5d8ce","rayCluster":"fail-hxn94"}
{"level":"error","ts":"2026-06-10T10:33:34.405Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a14c6ee3-f966-40b6-9477-dc63f9c5d8ce","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-hxn94\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:33:34.405Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a14c6ee3-f966-40b6-9477-dc63f9c5d8ce","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-hxn94\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.405Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a14c6ee3-f966-40b6-9477-dc63f9c5d8ce"}
{"level":"error","ts":"2026-06-10T10:33:34.405Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"a14c6ee3-f966-40b6-9477-dc63f9c5d8ce","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-hxn94\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.405Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"2a1f69af-e8c6-4061-b0ef-20f19e3a7f4a","DeletionTimestamp":"2026-06-10 10:33:34 +0000 UTC"}
{"level":"info","ts":"2026-06-10T10:33:34.405Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"2a1f69af-e8c6-4061-b0ef-20f19e3a7f4a","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:33:34.406Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"12415e07-464c-4e75-a5b4-b1660a2bc5d6","configMap":"kube-rbac-proxy-config-fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.412Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"12415e07-464c-4e75-a5b4-b1660a2bc5d6","serviceAccount":"fail-hxn94-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:34.415Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3a29af28-b751-4ef7-851d-f1b0f2211e8c","rayCluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.415Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3a29af28-b751-4ef7-851d-f1b0f2211e8c","rayCluster":"fail-hxn94"}
{"level":"error","ts":"2026-06-10T10:33:34.417Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3a29af28-b751-4ef7-851d-f1b0f2211e8c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-hxn94\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:33:34.417Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3a29af28-b751-4ef7-851d-f1b0f2211e8c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-hxn94\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.417Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3a29af28-b751-4ef7-851d-f1b0f2211e8c"}
{"level":"error","ts":"2026-06-10T10:33:34.417Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"3a29af28-b751-4ef7-851d-f1b0f2211e8c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-hxn94\" is forbidden: unable to create new content in namespace test-ns-bbzgk because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.417Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"be3a3e6e-7812-4b41-a0e4-5f570b06fa2d","DeletionTimestamp":"2026-06-10 10:33:34 +0000 UTC"}
{"level":"info","ts":"2026-06-10T10:33:34.418Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"be3a3e6e-7812-4b41-a0e4-5f570b06fa2d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:33:34.419Z","logger":"raycluster-resource","msg":"default","name":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.419Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-hxn94","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:33:34.428Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"12415e07-464c-4e75-a5b4-b1660a2bc5d6","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.428Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9e6b4f0a-b222-45f5-8421-776b208c3146","namespacedName":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:34.428Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9e6b4f0a-b222-45f5-8421-776b208c3146","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.428Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9e6b4f0a-b222-45f5-8421-776b208c3146","cluster":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.428Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9e6b4f0a-b222-45f5-8421-776b208c3146","namespace":"test-ns-bbzgk","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:33:34.428Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9e6b4f0a-b222-45f5-8421-776b208c3146","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:33:34.428Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cf35dd29-e4c7-4c61-82f5-059e5b246e62","namespace":"test-ns-bbzgk","clusterName":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.428Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"683b4e89-f230-416a-b4a1-ffb09e59b56d"}
{"level":"info","ts":"2026-06-10T10:33:34.429Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"a56bfe0c-b876-42a2-b4c7-65b1445f8950"}
{"level":"info","ts":"2026-06-10T10:33:34.429Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d9bcc0c2-003b-4283-98cd-4a690585c25d","DeletionTimestamp":"2026-06-10 10:33:34 +0000 UTC"}
{"level":"info","ts":"2026-06-10T10:33:34.429Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d9bcc0c2-003b-4283-98cd-4a690585c25d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-06-10T10:33:34.431Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9e6b4f0a-b222-45f5-8421-776b208c3146","error":"rayclusters.ray.io \"fail-hxn94\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.431Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9e6b4f0a-b222-45f5-8421-776b208c3146"}
{"level":"error","ts":"2026-06-10T10:33:34.431Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"9e6b4f0a-b222-45f5-8421-776b208c3146","error":"rayclusters.ray.io \"fail-hxn94\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.431Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1d4b57b9-90fd-4493-a78e-7769cfc0ffb1","namespacedName":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:34.431Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1d4b57b9-90fd-4493-a78e-7769cfc0ffb1"}
{"level":"info","ts":"2026-06-10T10:33:34.431Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1d4b57b9-90fd-4493-a78e-7769cfc0ffb1","namespace":"test-ns-bbzgk","authClusters":0}
{"level":"info","ts":"2026-06-10T10:33:34.431Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1d4b57b9-90fd-4493-a78e-7769cfc0ffb1","namespace":"test-ns-bbzgk"}
{"level":"error","ts":"2026-06-10T10:33:34.433Z","logger":"controllers.RayJob","msg":"Failed to remove finalizer for RayJob","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d9bcc0c2-003b-4283-98cd-4a690585c25d","error":"rayjobs.ray.io \"fail\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:130\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.433Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d9bcc0c2-003b-4283-98cd-4a690585c25d"}
{"level":"error","ts":"2026-06-10T10:33:34.433Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"d9bcc0c2-003b-4283-98cd-4a690585c25d","error":"rayjobs.ray.io \"fail\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.433Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"6dfbbb16-c2dd-47d4-aa91-0d53af0fe3c5"}
{"level":"info","ts":"2026-06-10T10:33:34.433Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1035fad2-2eda-4410-93fd-4d85727a2947","namespacedName":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:34.433Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1035fad2-2eda-4410-93fd-4d85727a2947"}
{"level":"info","ts":"2026-06-10T10:33:34.433Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1035fad2-2eda-4410-93fd-4d85727a2947","namespace":"test-ns-bbzgk","authClusters":0}
{"level":"info","ts":"2026-06-10T10:33:34.433Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"1035fad2-2eda-4410-93fd-4d85727a2947","namespace":"test-ns-bbzgk"}
{"level":"info","ts":"2026-06-10T10:33:34.499Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"e87e40ab-8e74-40b7-aa91-f3d0c1f273ee","namespacedName":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"}}
{"level":"info","ts":"2026-06-10T10:33:34.499Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-bbzgk"},"reconcileID":"b1b55ce5-7f4f-4f00-9daa-d930ee03cf31"}
{"level":"info","ts":"2026-06-10T10:33:34.499Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"e87e40ab-8e74-40b7-aa91-f3d0c1f273ee"}
{"level":"info","ts":"2026-06-10T10:33:34.499Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"e87e40ab-8e74-40b7-aa91-f3d0c1f273ee","namespace":"test-ns-bbzgk","authClusters":0}
{"level":"info","ts":"2026-06-10T10:33:34.499Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"e87e40ab-8e74-40b7-aa91-f3d0c1f273ee","namespace":"test-ns-bbzgk"}
{"level":"error","ts":"2026-06-10T10:33:34.502Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cf35dd29-e4c7-4c61-82f5-059e5b246e62","certificate":"ray-worker-cert-fail-hxn94","error":"certificates.cert-manager.io \"ray-worker-cert-fail-hxn94\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.519Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cf35dd29-e4c7-4c61-82f5-059e5b246e62","errorCount":1}
{"level":"info","ts":"2026-06-10T10:33:34.519Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cf35dd29-e4c7-4c61-82f5-059e5b246e62"}
{"level":"error","ts":"2026-06-10T10:33:34.519Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"cf35dd29-e4c7-4c61-82f5-059e5b246e62","error":"certificates.cert-manager.io \"ray-worker-cert-fail-hxn94\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:34.519Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"80174351-2435-47c2-84db-6320bdacb060","namespace":"test-ns-bbzgk","clusterName":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.519Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"80174351-2435-47c2-84db-6320bdacb060","namespace":"test-ns-bbzgk","clusterName":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.560Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"ecdc03c1-b92c-4783-a4b2-df7799bd6c5a","namespace":"test-ns-bbzgk","clusterName":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.560Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-hxn94","namespace":"test-ns-bbzgk"},"namespace":"test-ns-bbzgk","name":"fail-hxn94","reconcileID":"ecdc03c1-b92c-4783-a4b2-df7799bd6c5a","namespace":"test-ns-bbzgk","clusterName":"fail-hxn94"}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825"}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825"}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","podName":"long-running-gcmch-head-qr4ms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","podName":"long-running-gcmch-head-qr4ms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:33:34.715Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"2c2861c6-75e5-427e-9950-207dbc541825","name":"long-running-gcmch","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:34.728Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b523b7bb-cbbe-46ed-bef1-6df8f31413e9","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:34.728Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"be954a56-1045-4294-838f-824cd907e393","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:34.729Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b523b7bb-cbbe-46ed-bef1-6df8f31413e9","pod":"long-running-gcmch-head-qr4ms","ip":"10.134.0.19"}
{"level":"info","ts":"2026-06-10T10:33:34.729Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b523b7bb-cbbe-46ed-bef1-6df8f31413e9","pod":"long-running-gcmch-small-group-worker-tzxbm","ip":"10.133.0.15"}
{"level":"info","ts":"2026-06-10T10:33:34.729Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b523b7bb-cbbe-46ed-bef1-6df8f31413e9","rayCluster":"long-running-gcmch","podIPs":["10.134.0.19","10.133.0.15"]}
{"level":"info","ts":"2026-06-10T10:33:34.729Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c998045b-d843-40ee-9673-fca98e5f5bd0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:34.729Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c998045b-d843-40ee-9673-fca98e5f5bd0","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:34.729Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c998045b-d843-40ee-9673-fca98e5f5bd0","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:34.729Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c998045b-d843-40ee-9673-fca98e5f5bd0"}
{"level":"info","ts":"2026-06-10T10:33:34.729Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c998045b-d843-40ee-9673-fca98e5f5bd0","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:34.740Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"be954a56-1045-4294-838f-824cd907e393","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:34.750Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"be954a56-1045-4294-838f-824cd907e393","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:34.750Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"be954a56-1045-4294-838f-824cd907e393","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:36.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5401da05-5e4d-4172-ab1e-a42f550429c7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:36.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5401da05-5e4d-4172-ab1e-a42f550429c7","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:36.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5401da05-5e4d-4172-ab1e-a42f550429c7","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:36.303Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5401da05-5e4d-4172-ab1e-a42f550429c7"}
{"level":"info","ts":"2026-06-10T10:33:36.303Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5401da05-5e4d-4172-ab1e-a42f550429c7","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78"}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78"}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","podName":"long-running-gcmch-head-qr4ms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","podName":"long-running-gcmch-head-qr4ms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:36.729Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"7c0c448f-52e1-42bc-b32f-9445a6eadc78","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:37.418Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-bbzgk"},"reconcileID":"5c1f6e53-c589-48e1-9001-74b501e00144"}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83"}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83"}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","podName":"long-running-gcmch-head-qr4ms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:33:38.062Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","podName":"long-running-gcmch-head-qr4ms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:33:38.063Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:38.063Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"d4c116e2-54b8-4449-be43-18492a720f83","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:39.065Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15"}
{"level":"info","ts":"2026-06-10T10:33:39.065Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15"}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","podName":"long-running-gcmch-small-group-worker-tzxbm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","podName":"long-running-gcmch-small-group-worker-tzxbm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:33:39.066Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"25c60c34-269e-4075-bab4-3e29a75adb15","name":"long-running-gcmch","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"76cd8cf0-89f2-445b-938b-471daa7b67a6","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3b9d5e5b-c229-4088-8531-88f8aa8774ee","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a6cdf3c3-a803-446b-bba4-5af6cc540d91","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a6cdf3c3-a803-446b-bba4-5af6cc540d91","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a6cdf3c3-a803-446b-bba4-5af6cc540d91","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3b9d5e5b-c229-4088-8531-88f8aa8774ee","pod":"long-running-gcmch-head-qr4ms","ip":"10.134.0.19"}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3b9d5e5b-c229-4088-8531-88f8aa8774ee","pod":"long-running-gcmch-small-group-worker-tzxbm","ip":"10.133.0.15"}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3b9d5e5b-c229-4088-8531-88f8aa8774ee","rayCluster":"long-running-gcmch","podIPs":["10.134.0.19","10.133.0.15"]}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a6cdf3c3-a803-446b-bba4-5af6cc540d91"}
{"level":"info","ts":"2026-06-10T10:33:39.079Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a6cdf3c3-a803-446b-bba4-5af6cc540d91","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:39.090Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"76cd8cf0-89f2-445b-938b-471daa7b67a6","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:39.098Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"76cd8cf0-89f2-445b-938b-471daa7b67a6","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:39.098Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"76cd8cf0-89f2-445b-938b-471daa7b67a6","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:39.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ba3c5358-76f5-45ac-83f3-248360697e39","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:39.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ba3c5358-76f5-45ac-83f3-248360697e39","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:39.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ba3c5358-76f5-45ac-83f3-248360697e39","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:39.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ba3c5358-76f5-45ac-83f3-248360697e39"}
{"level":"info","ts":"2026-06-10T10:33:39.304Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ba3c5358-76f5-45ac-83f3-248360697e39","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6"}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6"}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","podName":"long-running-gcmch-head-qr4ms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","podName":"long-running-gcmch-head-qr4ms","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:41.079Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"a4d52377-0a21-41ad-9fcd-ad37d2e150e6","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:42.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9ac722f6-ff3d-4e3a-ae44-f506d5d7e277","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:42.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9ac722f6-ff3d-4e3a-ae44-f506d5d7e277","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:42.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9ac722f6-ff3d-4e3a-ae44-f506d5d7e277","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:42.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9ac722f6-ff3d-4e3a-ae44-f506d5d7e277"}
{"level":"info","ts":"2026-06-10T10:33:42.304Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9ac722f6-ff3d-4e3a-ae44-f506d5d7e277","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:45.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c2c35c72-3e4c-42b6-b84b-90acd463df74","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:45.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c2c35c72-3e4c-42b6-b84b-90acd463df74","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:45.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c2c35c72-3e4c-42b6-b84b-90acd463df74","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:45.305Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c2c35c72-3e4c-42b6-b84b-90acd463df74"}
{"level":"info","ts":"2026-06-10T10:33:45.305Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c2c35c72-3e4c-42b6-b84b-90acd463df74","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:45.750Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1"}
{"level":"info","ts":"2026-06-10T10:33:45.750Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1"}
{"level":"info","ts":"2026-06-10T10:33:45.750Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","podName":"long-running-gcmch-small-group-worker-tzxbm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","podName":"long-running-gcmch-small-group-worker-tzxbm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:33:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:33:45.751Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"1be038ae-8442-4e15-860b-f3b83b1910f1","name":"long-running-gcmch","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:33:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"e7df44e5-4a7f-4838-ab7c-98a97b95f970","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84576f17-7e62-4b5c-9a04-643b27f1b518","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84576f17-7e62-4b5c-9a04-643b27f1b518","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fa4367ea-ae17-4b2e-86a3-d426d4296af5","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84576f17-7e62-4b5c-9a04-643b27f1b518","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"e7df44e5-4a7f-4838-ab7c-98a97b95f970","pod":"long-running-gcmch-head-qr4ms","ip":"10.134.0.19"}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"e7df44e5-4a7f-4838-ab7c-98a97b95f970","pod":"long-running-gcmch-small-group-worker-tzxbm","ip":"10.133.0.15"}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"e7df44e5-4a7f-4838-ab7c-98a97b95f970","rayCluster":"long-running-gcmch","podIPs":["10.134.0.19","10.133.0.15"]}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84576f17-7e62-4b5c-9a04-643b27f1b518"}
{"level":"info","ts":"2026-06-10T10:33:45.768Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84576f17-7e62-4b5c-9a04-643b27f1b518","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:45.780Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fa4367ea-ae17-4b2e-86a3-d426d4296af5","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:45.794Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fa4367ea-ae17-4b2e-86a3-d426d4296af5","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:45.794Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"fa4367ea-ae17-4b2e-86a3-d426d4296af5","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:46.910Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a010a13a-86f9-47db-95ac-6b3c1476108a","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:33:46.910Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a010a13a-86f9-47db-95ac-6b3c1476108a","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:33:46.910Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a010a13a-86f9-47db-95ac-6b3c1476108a","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:33:46.911Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"a010a13a-86f9-47db-95ac-6b3c1476108a","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4"}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4"}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","podName":"long-running-gcmch-small-group-worker-tzxbm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","podName":"long-running-gcmch-small-group-worker-tzxbm","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:27Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:47.769Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"46bdafc6-ae82-4160-8289-539ea2858cb4","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:48.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ff5cfb-5a9d-4f34-a087-b0b96afce928","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:48.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ff5cfb-5a9d-4f34-a087-b0b96afce928","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:48.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ff5cfb-5a9d-4f34-a087-b0b96afce928","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:48.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ff5cfb-5a9d-4f34-a087-b0b96afce928"}
{"level":"info","ts":"2026-06-10T10:33:48.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ff5cfb-5a9d-4f34-a087-b0b96afce928","RayCluster":"long-running-gcmch","State":""}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3"}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3"}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","1 head service found":"long-running-gcmch-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","Found 1 head Pod":"long-running-gcmch-head-qr4ms","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","head Pod":"long-running-gcmch-head-qr4ms","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-gcmch-head-qr4ms. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","worker Pod":"long-running-gcmch-small-group-worker-tzxbm","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-gcmch-small-group-worker-tzxbm. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:33:50.093Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"23047929-9670-4397-8047-fd66e2d4e9c3","name":"long-running-gcmch","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:33:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:50Z","stateTransitionTimes":{"ready":"2026-06-10T10:33:50Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.19","serviceIP":"10.134.0.19","podName":"long-running-gcmch-head-qr4ms","serviceName":"long-running-gcmch-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:33:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:33:50Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:24Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:24Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"ccf0f800-04d1-421c-a778-e4997bf4ff95","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"6a455cf9-763f-4666-a517-5fc49979c8ed","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"ccf0f800-04d1-421c-a778-e4997bf4ff95","pod":"long-running-gcmch-head-qr4ms","ip":"10.134.0.19"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"ccf0f800-04d1-421c-a778-e4997bf4ff95","pod":"long-running-gcmch-small-group-worker-tzxbm","ip":"10.133.0.15"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"ccf0f800-04d1-421c-a778-e4997bf4ff95","rayCluster":"long-running-gcmch","podIPs":["10.134.0.19","10.133.0.15"]}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","head service name":"long-running-gcmch-head-svc","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","head service URL":"long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552"}
{"level":"info","ts":"2026-06-10T10:33:50.106Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","command":["if","!","ray","job","status","--address","http://long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265","long-running-sfldr",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265","--no-wait","--submission-id","long-running-sfldr","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265","--follow","long-running-sfldr"]}
{"level":"info","ts":"2026-06-10T10:33:50.118Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-10T10:33:50.118Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","SubmissionMode":"K8sJobMode","RayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.118Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfldr","rayClusterName":"long-running-gcmch","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfldr","rayClusterName":"long-running-gcmch","dashboardURL":"long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:33:50.118Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"84ace8ed-1847-4399-b708-3db19f409552","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:33:50.123Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"6a455cf9-763f-4666-a517-5fc49979c8ed","name":"long-running-gcmch-head"}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e"}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","head service name":"long-running-gcmch-head-svc","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","head service URL":"long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","SubmissionMode":"K8sJobMode","RayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfldr","rayClusterName":"long-running-gcmch","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfldr","rayClusterName":"long-running-gcmch","dashboardURL":"long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:33:50.131Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:33:50.140Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"6a455cf9-763f-4666-a517-5fc49979c8ed","name":"long-running-gcmch-workers"}
{"level":"info","ts":"2026-06-10T10:33:50.140Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"6a455cf9-763f-4666-a517-5fc49979c8ed","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.141Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-10T10:33:50.141Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e"}
{"level":"error","ts":"2026-06-10T10:33:50.141Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"323bf255-bfdd-47b2-8774-d1a0a1cd198e","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.141Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d4e09a2f-23e9-4b2b-8b26-9fb22e3b42e2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.141Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d4e09a2f-23e9-4b2b-8b26-9fb22e3b42e2","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.141Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d4e09a2f-23e9-4b2b-8b26-9fb22e3b42e2","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.141Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d4e09a2f-23e9-4b2b-8b26-9fb22e3b42e2"}
{"level":"error","ts":"2026-06-10T10:33:50.150Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d4e09a2f-23e9-4b2b-8b26-9fb22e3b42e2","JobId":"long-running-sfldr","error":"Job long-running-sfldr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.150Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d4e09a2f-23e9-4b2b-8b26-9fb22e3b42e2"}
{"level":"error","ts":"2026-06-10T10:33:50.150Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d4e09a2f-23e9-4b2b-8b26-9fb22e3b42e2","error":"Job long-running-sfldr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.150Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e578d02a-8ce9-4435-83e8-3c4dac558a94","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.150Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e578d02a-8ce9-4435-83e8-3c4dac558a94","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.150Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e578d02a-8ce9-4435-83e8-3c4dac558a94","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.150Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e578d02a-8ce9-4435-83e8-3c4dac558a94"}
{"level":"error","ts":"2026-06-10T10:33:50.155Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e578d02a-8ce9-4435-83e8-3c4dac558a94","JobId":"long-running-sfldr","error":"Job long-running-sfldr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.155Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e578d02a-8ce9-4435-83e8-3c4dac558a94"}
{"level":"error","ts":"2026-06-10T10:33:50.155Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e578d02a-8ce9-4435-83e8-3c4dac558a94","error":"Job long-running-sfldr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.161Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4a99fd39-c20a-45d0-9321-7d792ae89096","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.161Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4a99fd39-c20a-45d0-9321-7d792ae89096","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.161Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4a99fd39-c20a-45d0-9321-7d792ae89096","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.161Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4a99fd39-c20a-45d0-9321-7d792ae89096"}
{"level":"error","ts":"2026-06-10T10:33:50.207Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4a99fd39-c20a-45d0-9321-7d792ae89096","JobId":"long-running-sfldr","error":"Job long-running-sfldr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.208Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4a99fd39-c20a-45d0-9321-7d792ae89096"}
{"level":"error","ts":"2026-06-10T10:33:50.208Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4a99fd39-c20a-45d0-9321-7d792ae89096","error":"Job long-running-sfldr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.248Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4ab8f08a-c759-425a-a347-d9d6ff42ea87","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.248Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4ab8f08a-c759-425a-a347-d9d6ff42ea87","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.248Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4ab8f08a-c759-425a-a347-d9d6ff42ea87","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.248Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4ab8f08a-c759-425a-a347-d9d6ff42ea87"}
{"level":"error","ts":"2026-06-10T10:33:50.253Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4ab8f08a-c759-425a-a347-d9d6ff42ea87","JobId":"long-running-sfldr","error":"Job long-running-sfldr does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.253Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4ab8f08a-c759-425a-a347-d9d6ff42ea87"}
{"level":"error","ts":"2026-06-10T10:33:50.253Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4ab8f08a-c759-425a-a347-d9d6ff42ea87","error":"Job long-running-sfldr does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.296Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b84b2957-14c3-417e-92c8-5911adfd630c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.297Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b84b2957-14c3-417e-92c8-5911adfd630c","oldStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:33:50.297Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b84b2957-14c3-417e-92c8-5911adfd630c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfldr","rayClusterName":"long-running-gcmch","dashboardURL":"long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfldr","rayClusterName":"long-running-gcmch","dashboardURL":"long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:33:50.297Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b84b2957-14c3-417e-92c8-5911adfd630c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-06-10T10:33:50.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c7e939e1-eeb2-4633-a4b4-4836a7dc7c99","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.316Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c7e939e1-eeb2-4633-a4b4-4836a7dc7c99","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.316Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c7e939e1-eeb2-4633-a4b4-4836a7dc7c99","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"e5606bc8-159a-4f83-9875-9b5ba177f237","rayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5c3cb5e0-2156-48ce-9257-c4ed5626bd53","namespacedName":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"reconcileID":"0a42d0e2-2162-4945-8a0f-69557e4f812c"}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5c3cb5e0-2156-48ce-9257-c4ed5626bd53","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5c3cb5e0-2156-48ce-9257-c4ed5626bd53","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"6a3efd7e-e244-419f-b477-0c41ef44fe0c"}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"e5606bc8-159a-4f83-9875-9b5ba177f237","pod":"long-running-gcmch-head-qr4ms","ip":"10.134.0.19"}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"e5606bc8-159a-4f83-9875-9b5ba177f237","pod":"long-running-gcmch-small-group-worker-tzxbm","ip":"10.133.0.15"}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5c3cb5e0-2156-48ce-9257-c4ed5626bd53","namespace":"test-ns-nrv9s","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:33:50.317Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"e5606bc8-159a-4f83-9875-9b5ba177f237","rayCluster":"long-running-gcmch","podIPs":["10.134.0.19","10.133.0.15"]}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"controllers.RayJob","msg":"The associated submitter Kubernetes Job for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c7e939e1-eeb2-4633-a4b4-4836a7dc7c99","Submitter K8s Job":"long-running"}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5c3cb5e0-2156-48ce-9257-c4ed5626bd53","name":"kuberay-gateway-access","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c7e939e1-eeb2-4633-a4b4-4836a7dc7c99","isJobDeleted":false}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"c7e939e1-eeb2-4633-a4b4-4836a7dc7c99"}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"adbf9126-5a2f-4a94-ae07-ca1b6ff4201d","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"adbf9126-5a2f-4a94-ae07-ca1b6ff4201d","RayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"adbf9126-5a2f-4a94-ae07-ca1b6ff4201d","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"adbf9126-5a2f-4a94-ae07-ca1b6ff4201d","Kubernetes Job":""}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"adbf9126-5a2f-4a94-ae07-ca1b6ff4201d","isJobDeleted":true}
{"level":"info","ts":"2026-06-10T10:33:50.324Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"adbf9126-5a2f-4a94-ae07-ca1b6ff4201d"}
{"level":"info","ts":"2026-06-10T10:33:50.329Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5c3cb5e0-2156-48ce-9257-c4ed5626bd53","name":"test-ns-nrv9s-long-running-gcmch","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:33:50.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4f1f0a08-f5d7-45a5-817e-05a72d466056","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.334Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4f1f0a08-f5d7-45a5-817e-05a72d466056","RayCluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.334Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4f1f0a08-f5d7-45a5-817e-05a72d466056","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:33:50.334Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4f1f0a08-f5d7-45a5-817e-05a72d466056","Kubernetes Job":""}
{"level":"info","ts":"2026-06-10T10:33:50.334Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4f1f0a08-f5d7-45a5-817e-05a72d466056","isJobDeleted":true}
{"level":"info","ts":"2026-06-10T10:33:50.334Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"4f1f0a08-f5d7-45a5-817e-05a72d466056"}
{"level":"info","ts":"2026-06-10T10:33:50.336Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5c3cb5e0-2156-48ce-9257-c4ed5626bd53","configMap":"kube-rbac-proxy-config-long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.343Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5c3cb5e0-2156-48ce-9257-c4ed5626bd53","serviceAccount":"long-running-gcmch-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:50.350Z","logger":"raycluster-resource","msg":"default","name":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.350Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-gcmch","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:50.361Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5c3cb5e0-2156-48ce-9257-c4ed5626bd53","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.361Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"45c14ef0-455b-4321-8239-c6383b9b6469","namespacedName":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.361Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"45c14ef0-455b-4321-8239-c6383b9b6469","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.361Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"45c14ef0-455b-4321-8239-c6383b9b6469","cluster":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.361Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"45c14ef0-455b-4321-8239-c6383b9b6469","namespace":"test-ns-nrv9s","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:33:50.361Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"45c14ef0-455b-4321-8239-c6383b9b6469","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:33:50.361Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3f6e4d71-7736-481c-83b8-7d0bc08b75d2","namespace":"test-ns-nrv9s","clusterName":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.361Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"0a9fd525-a4b1-4e91-ad10-f9d196b2a7df"}
{"level":"info","ts":"2026-06-10T10:33:50.362Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"56c4088a-064a-4111-9c50-64a48a298329","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.362Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"56c4088a-064a-4111-9c50-64a48a298329","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.362Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"56c4088a-064a-4111-9c50-64a48a298329","isClusterDeleted":true}
{"level":"info","ts":"2026-06-10T10:33:50.362Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"56c4088a-064a-4111-9c50-64a48a298329","Kubernetes Job":""}
{"level":"info","ts":"2026-06-10T10:33:50.362Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"56c4088a-064a-4111-9c50-64a48a298329","isJobDeleted":true}
{"level":"info","ts":"2026-06-10T10:33:50.362Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"56c4088a-064a-4111-9c50-64a48a298329","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-sfldr","rayClusterName":"long-running-gcmch","dashboardURL":"long-running-gcmch-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"error","ts":"2026-06-10T10:33:50.365Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"45c14ef0-455b-4321-8239-c6383b9b6469","error":"rayclusters.ray.io \"long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.365Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"45c14ef0-455b-4321-8239-c6383b9b6469"}
{"level":"error","ts":"2026-06-10T10:33:50.365Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"45c14ef0-455b-4321-8239-c6383b9b6469","error":"rayclusters.ray.io \"long-running-gcmch\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.365Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b31c32fc-9a50-4570-841b-7ec77a4592a3","namespacedName":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.365Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b31c32fc-9a50-4570-841b-7ec77a4592a3"}
{"level":"info","ts":"2026-06-10T10:33:50.365Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b31c32fc-9a50-4570-841b-7ec77a4592a3","namespace":"test-ns-nrv9s","authClusters":0}
{"level":"info","ts":"2026-06-10T10:33:50.365Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b31c32fc-9a50-4570-841b-7ec77a4592a3","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:50.362Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"56c4088a-064a-4111-9c50-64a48a298329","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-06-10T10:33:50.399Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b392d1f-5807-4c1b-8f53-4808c46b8dcd","namespacedName":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.399Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b392d1f-5807-4c1b-8f53-4808c46b8dcd"}
{"level":"info","ts":"2026-06-10T10:33:50.399Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b392d1f-5807-4c1b-8f53-4808c46b8dcd","namespace":"test-ns-nrv9s","authClusters":0}
{"level":"info","ts":"2026-06-10T10:33:50.399Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b392d1f-5807-4c1b-8f53-4808c46b8dcd","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:50.399Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"dd622db3-9980-4230-931c-5392a065304f"}
{"level":"info","ts":"2026-06-10T10:33:50.399Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4d7d0117-6c1f-4f8d-8f2e-b6c6c25ca324","namespacedName":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:50.399Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4d7d0117-6c1f-4f8d-8f2e-b6c6c25ca324"}
{"level":"info","ts":"2026-06-10T10:33:50.399Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4d7d0117-6c1f-4f8d-8f2e-b6c6c25ca324","namespace":"test-ns-nrv9s","authClusters":0}
{"level":"info","ts":"2026-06-10T10:33:50.399Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4d7d0117-6c1f-4f8d-8f2e-b6c6c25ca324","namespace":"test-ns-nrv9s"}
{"level":"error","ts":"2026-06-10T10:33:50.403Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3f6e4d71-7736-481c-83b8-7d0bc08b75d2","certificate":"ray-head-cert-long-running-gcmch","error":"certificates.cert-manager.io \"ray-head-cert-long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:33:50.406Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3f6e4d71-7736-481c-83b8-7d0bc08b75d2","certificate":"ray-ca-certificate-long-running-gcmch","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.412Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"8b3a97b6-2cea-4bfb-9eac-c2da3ff89525","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:50.422Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3f6e4d71-7736-481c-83b8-7d0bc08b75d2","errorCount":2}
{"level":"info","ts":"2026-06-10T10:33:50.422Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3f6e4d71-7736-481c-83b8-7d0bc08b75d2"}
{"level":"error","ts":"2026-06-10T10:33:50.422Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"3f6e4d71-7736-481c-83b8-7d0bc08b75d2","error":"certificates.cert-manager.io \"ray-head-cert-long-running-gcmch\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.427Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b680cbe-6684-400f-b7d0-99b2abbab096","namespace":"test-ns-nrv9s","clusterName":"long-running-gcmch"}
{"level":"error","ts":"2026-06-10T10:33:50.430Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b680cbe-6684-400f-b7d0-99b2abbab096","certificate":"ray-head-cert-long-running-gcmch","error":"certificates.cert-manager.io \"ray-head-cert-long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:33:50.433Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b680cbe-6684-400f-b7d0-99b2abbab096","certificate":"ray-ca-certificate-long-running-gcmch","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.433Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b680cbe-6684-400f-b7d0-99b2abbab096","errorCount":2}
{"level":"info","ts":"2026-06-10T10:33:50.433Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b680cbe-6684-400f-b7d0-99b2abbab096"}
{"level":"error","ts":"2026-06-10T10:33:50.433Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"4b680cbe-6684-400f-b7d0-99b2abbab096","error":"certificates.cert-manager.io \"ray-head-cert-long-running-gcmch\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.443Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b78efe9a-2f2d-42e1-998d-82aa0741d915","namespace":"test-ns-nrv9s","clusterName":"long-running-gcmch"}
{"level":"error","ts":"2026-06-10T10:33:50.447Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b78efe9a-2f2d-42e1-998d-82aa0741d915","certificate":"ray-head-cert-long-running-gcmch","error":"certificates.cert-manager.io \"ray-head-cert-long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:33:50.450Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b78efe9a-2f2d-42e1-998d-82aa0741d915","certificate":"ray-ca-certificate-long-running-gcmch","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.450Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b78efe9a-2f2d-42e1-998d-82aa0741d915","errorCount":2}
{"level":"info","ts":"2026-06-10T10:33:50.450Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b78efe9a-2f2d-42e1-998d-82aa0741d915"}
{"level":"error","ts":"2026-06-10T10:33:50.450Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"b78efe9a-2f2d-42e1-998d-82aa0741d915","error":"certificates.cert-manager.io \"ray-head-cert-long-running-gcmch\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.470Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"de2089e3-1673-408c-a853-73dd8e166150","namespace":"test-ns-nrv9s","clusterName":"long-running-gcmch"}
{"level":"error","ts":"2026-06-10T10:33:50.473Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"de2089e3-1673-408c-a853-73dd8e166150","certificate":"ray-head-cert-long-running-gcmch","error":"certificates.cert-manager.io \"ray-head-cert-long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:33:50.476Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"de2089e3-1673-408c-a853-73dd8e166150","certificate":"ray-ca-certificate-long-running-gcmch","error":"certificates.cert-manager.io \"ray-ca-certificate-long-running-gcmch\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.477Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"de2089e3-1673-408c-a853-73dd8e166150","errorCount":2}
{"level":"info","ts":"2026-06-10T10:33:50.477Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"de2089e3-1673-408c-a853-73dd8e166150"}
{"level":"error","ts":"2026-06-10T10:33:50.477Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"de2089e3-1673-408c-a853-73dd8e166150","error":"certificates.cert-manager.io \"ray-head-cert-long-running-gcmch\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:50.517Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5cc1132e-6482-4fda-9ce0-904ff53ef38e","namespace":"test-ns-nrv9s","clusterName":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:50.517Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-gcmch","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-gcmch","reconcileID":"5cc1132e-6482-4fda-9ce0-904ff53ef38e","namespace":"test-ns-nrv9s","clusterName":"long-running-gcmch"}
{"level":"info","ts":"2026-06-10T10:33:51.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"21ef99ff-4330-4852-afd0-ac627317dd18","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:51.326Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"21ef99ff-4330-4852-afd0-ac627317dd18"}
{"level":"info","ts":"2026-06-10T10:33:51.326Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"21ef99ff-4330-4852-afd0-ac627317dd18","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:33:51.326Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"21ef99ff-4330-4852-afd0-ac627317dd18","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-06-10T10:33:51.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b93174a8-6436-4859-bf83-dad3d246b73f","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:51.336Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b93174a8-6436-4859-bf83-dad3d246b73f"}
{"level":"info","ts":"2026-06-10T10:33:51.336Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b93174a8-6436-4859-bf83-dad3d246b73f","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:33:51.336Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b93174a8-6436-4859-bf83-dad3d246b73f","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-10T10:33:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ng82r","rayClusterName":"long-running-scs76","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:33:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:33:51.336Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b93174a8-6436-4859-bf83-dad3d246b73f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:33:51.346Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"067cd824-2c76-4661-b0df-9ee3e49ff20e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:51.346Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"067cd824-2c76-4661-b0df-9ee3e49ff20e","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.346Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"067cd824-2c76-4661-b0df-9ee3e49ff20e","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.346Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"067cd824-2c76-4661-b0df-9ee3e49ff20e","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.351Z","logger":"raycluster-resource","msg":"default","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.351Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-scs76","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"067cd824-2c76-4661-b0df-9ee3e49ff20e","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"a783759a-57ff-4eea-a3e3-58d2d85c676c","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","namespacedName":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5"}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5"}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","mode":"IntegratedOAuth","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.360Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.361Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"067cd824-2c76-4661-b0df-9ee3e49ff20e"}
{"level":"info","ts":"2026-06-10T10:33:51.361Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"067cd824-2c76-4661-b0df-9ee3e49ff20e","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:51.361Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7d12998e-5701-4030-9c30-5c74250448cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:51.361Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7d12998e-5701-4030-9c30-5c74250448cb","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.361Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7d12998e-5701-4030-9c30-5c74250448cb","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.361Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7d12998e-5701-4030-9c30-5c74250448cb"}
{"level":"info","ts":"2026-06-10T10:33:51.361Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7d12998e-5701-4030-9c30-5c74250448cb","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:51.499Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.499Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","name":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:51.499Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.499Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","error":"head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"error","ts":"2026-06-10T10:33:51.499Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.500Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:51.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.500Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:33:51.500Z","logger":"raycluster-resource","msg":"default","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.500Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","name":"long-running-scs76","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:51.500Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-scs76","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:33:51.509Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"a783759a-57ff-4eea-a3e3-58d2d85c676c","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.509Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"64cd7757-327c-4fac-8acb-da644c8c1e00","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.509Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"38ee1b0d-50ef-4609-8f60-50ed9764a30c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:51.509Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"38ee1b0d-50ef-4609-8f60-50ed9764a30c","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.509Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"38ee1b0d-50ef-4609-8f60-50ed9764a30c","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"38ee1b0d-50ef-4609-8f60-50ed9764a30c"}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"38ee1b0d-50ef-4609-8f60-50ed9764a30c","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","name":"long-running-scs76-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","name":"long-running-scs76","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-scs76\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"long-running-scs76"}}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5"}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb"}
{"level":"error","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"62bae92f-d089-41fe-9a9a-4302ac6fb8a5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c"}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c"}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","error":"head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"error","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:33:51.599Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","name":"long-running-scs76","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:51.600Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb","rayCluster":"long-running-scs76","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:33:51.600Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.699Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"64cd7757-327c-4fac-8acb-da644c8c1e00","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:33:51.699Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","name":"kuberay-gateway-access","namespace":"test-ns-nrv9s","operation":"created"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c"}
{"level":"error","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ed8955a6-594a-4ea3-804b-28993d0c2f6c","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"0b7435aa-6a1e-4de2-a95a-d3a8780bedf2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"0b7435aa-6a1e-4de2-a95a-d3a8780bedf2","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"0b7435aa-6a1e-4de2-a95a-d3a8780bedf2","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3","error":"head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"0b7435aa-6a1e-4de2-a95a-d3a8780bedf2"}
{"level":"error","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3"}
{"level":"error","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ec752aef-9360-48de-879c-22375538f1e3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.700Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"0b7435aa-6a1e-4de2-a95a-d3a8780bedf2","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:51.707Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","name":"test-ns-nrv9s-long-running-scs76","operation":"created"}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980"}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980"}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980","error":"head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb"}
{"level":"error","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb","rayCluster":"long-running-scs76","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980"}
{"level":"error","ts":"2026-06-10T10:33:51.710Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ce5983cd-72e3-4636-b70a-f83a1ecd8980","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.712Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","name":"kube-rbac-proxy-config-long-running-scs76","operation":"created"}
{"level":"info","ts":"2026-06-10T10:33:51.713Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"64cd7757-327c-4fac-8acb-da644c8c1e00","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:33:51.713Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"64cd7757-327c-4fac-8acb-da644c8c1e00","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.713Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"c75f6345-2ddc-41d9-a4c7-b074e3d3aa95","cluster":"long-running-scs76"}
{"level":"error","ts":"2026-06-10T10:33:51.717Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-scs76\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.717Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc"}
{"level":"error","ts":"2026-06-10T10:33:51.717Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"292384c3-0fee-4016-bc12-29a4420c30bc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-scs76\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.717Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"24d9b8a8-7bc1-40be-a875-c0f4124e26b5","namespacedName":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.718Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"24d9b8a8-7bc1-40be-a875-c0f4124e26b5","mode":"IntegratedOAuth","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.718Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"24d9b8a8-7bc1-40be-a875-c0f4124e26b5","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.721Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb"}
{"level":"info","ts":"2026-06-10T10:33:51.721Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb","rayCluster":"long-running-scs76","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:33:51.721Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5ea03cf3-2410-4d35-988c-1d3bb96b7efb"}
{"level":"info","ts":"2026-06-10T10:33:51.721Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"a560bf59-7300-40af-bc64-b6408a7ac35b","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.721Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"a560bf59-7300-40af-bc64-b6408a7ac35b"}
{"level":"info","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29"}
{"level":"info","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29"}
{"level":"info","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29","error":"head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"error","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29"}
{"level":"error","ts":"2026-06-10T10:33:51.799Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"236b8329-d4a4-4e86-9410-7949303c7d29","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"24d9b8a8-7bc1-40be-a875-c0f4124e26b5","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"811b3c4b-0118-456f-aa46-017b3e3df9e3","namespacedName":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"811b3c4b-0118-456f-aa46-017b3e3df9e3","mode":"IntegratedOAuth","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"811b3c4b-0118-456f-aa46-017b3e3df9e3","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"4a025b1d-5caf-4731-9dd1-45f8675a40df","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"4a025b1d-5caf-4731-9dd1-45f8675a40df"}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"42cda471-005f-4116-93ba-08937a86fcf5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"42cda471-005f-4116-93ba-08937a86fcf5","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"42cda471-005f-4116-93ba-08937a86fcf5","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"42cda471-005f-4116-93ba-08937a86fcf5"}
{"level":"info","ts":"2026-06-10T10:33:51.800Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"42cda471-005f-4116-93ba-08937a86fcf5","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:51.804Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"c75f6345-2ddc-41d9-a4c7-b074e3d3aa95","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f"}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f"}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f","error":"head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"error","ts":"2026-06-10T10:33:51.899Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"811b3c4b-0118-456f-aa46-017b3e3df9e3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-scs76\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"811b3c4b-0118-456f-aa46-017b3e3df9e3"}
{"level":"error","ts":"2026-06-10T10:33:51.899Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"811b3c4b-0118-456f-aa46-017b3e3df9e3","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-scs76\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f"}
{"level":"error","ts":"2026-06-10T10:33:51.899Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fa499392-d9f5-4627-8673-80641977520f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:51.903Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"c75f6345-2ddc-41d9-a4c7-b074e3d3aa95","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:33:51.903Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"c75f6345-2ddc-41d9-a4c7-b074e3d3aa95","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.904Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"7f9c0b26-776e-4c6c-a2aa-c2c9d3e7fb2b","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.904Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"49d58d59-f737-4bac-b27d-4e5e2ba5459d","namespacedName":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:51.905Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"49d58d59-f737-4bac-b27d-4e5e2ba5459d","mode":"IntegratedOAuth","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.905Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"49d58d59-f737-4bac-b27d-4e5e2ba5459d","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.916Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"49d58d59-f737-4bac-b27d-4e5e2ba5459d","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:51.918Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"7f9c0b26-776e-4c6c-a2aa-c2c9d3e7fb2b","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:33:51.928Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"7f9c0b26-776e-4c6c-a2aa-c2c9d3e7fb2b","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:33:51.928Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"7f9c0b26-776e-4c6c-a2aa-c2c9d3e7fb2b","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d"}
{"level":"info","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d"}
{"level":"info","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d","error":"head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"error","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d"}
{"level":"error","ts":"2026-06-10T10:33:52.060Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b0a199c6-60ea-400c-a77e-62daff68ab1d","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:52.381Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601"}
{"level":"info","ts":"2026-06-10T10:33:52.381Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601"}
{"level":"info","ts":"2026-06-10T10:33:52.381Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:52.381Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601","error":"head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"error","ts":"2026-06-10T10:33:52.381Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:52.381Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:52.381Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:52.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:52.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:52.382Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601"}
{"level":"error","ts":"2026-06-10T10:33:52.382Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bd797e1c-ae0d-4a12-a0eb-6d2eacf52601","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6"}
{"level":"info","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6"}
{"level":"info","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6","error":"head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"error","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found"}
{"level":"info","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"long-running-scs76","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6"}
{"level":"error","ts":"2026-06-10T10:33:53.023Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"63fac3ea-eff6-4ff9-9596-ca0581d255f6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-scs76 not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4"}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4"}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:53.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:53.281Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:53.281Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"177ada57-88cd-4974-a1a5-7436a28d58c4","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:53.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7afb6ff8-f0df-4ddd-a879-c42c79423d97","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:53.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7afb6ff8-f0df-4ddd-a879-c42c79423d97","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:53.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7afb6ff8-f0df-4ddd-a879-c42c79423d97","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:53.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7afb6ff8-f0df-4ddd-a879-c42c79423d97"}
{"level":"info","ts":"2026-06-10T10:33:53.335Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7afb6ff8-f0df-4ddd-a879-c42c79423d97","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0"}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0"}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0"}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0"}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","mode":"IntegratedOAuth","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","cluster":"long-running-scs76","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","cluster":"long-running-scs76","authType":"IntegratedOAuth","serviceAccount":"long-running-scs76-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-scs76","ray.io/group":"headgroup","ray.io/identifier":"long-running-scs76-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:33:54.304Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","enabled":true}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","name":"long-running-scs76-head-cfpwz"}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0"}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","nodeType":"worker","rayStartParams":{"address":"long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","rayStartCmd":"ray start  --address=long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:33:54.319Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","enabled":true}
{"level":"info","ts":"2026-06-10T10:33:54.331Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","name":"long-running-scs76-small-group-worker-4pdq5"}
{"level":"info","ts":"2026-06-10T10:33:54.331Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.331Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-scs76-head-svc"},"newHead":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"}}
{"level":"info","ts":"2026-06-10T10:33:54.331Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"bcbd90b0-5a8d-49b9-be83-e7048353eca0","name":"long-running-scs76","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6714292f-c5f2-47de-b39b-780c2c9ec094","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"40591653-a789-4052-9094-e15c6dbb9cbe","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f49ce0be-0e33-4060-ab54-5649fb4e1032","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f49ce0be-0e33-4060-ab54-5649fb4e1032","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f49ce0be-0e33-4060-ab54-5649fb4e1032","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"40591653-a789-4052-9094-e15c6dbb9cbe","rayCluster":"long-running-scs76","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f49ce0be-0e33-4060-ab54-5649fb4e1032"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f49ce0be-0e33-4060-ab54-5649fb4e1032","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-scs76-head-svc"},"newHead":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"}}
{"level":"info","ts":"2026-06-10T10:33:54.345Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","name":"long-running-scs76","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","name":"long-running-scs76","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-scs76\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"long-running-scs76"}}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f"}
{"level":"error","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"76a67afa-6ea1-44a4-8195-e6fcd06abb3f","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-scs76\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27"}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27"}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:54.353Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"c7bb5f27-6690-4231-aaa9-c9c7a32a6d27","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6714292f-c5f2-47de-b39b-780c2c9ec094","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"fda086b0-8280-48d3-a937-087ae205ecca","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:54.359Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:54.371Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6714292f-c5f2-47de-b39b-780c2c9ec094","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6714292f-c5f2-47de-b39b-780c2c9ec094","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:33:54.399Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"a44f3487-145a-4ca4-a944-9cb3896286b0","name":"long-running-scs76","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8"}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8"}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.410Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"c7531c01-dfae-4694-9ece-ad5b957c1642","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"21a0b02c-a3ed-48c3-9c99-48a7766c527e","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"21a0b02c-a3ed-48c3-9c99-48a7766c527e","rayCluster":"long-running-scs76","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7239c70d-39cc-4030-abdf-070b252f88b3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","name":"long-running-scs76","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7239c70d-39cc-4030-abdf-070b252f88b3","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7239c70d-39cc-4030-abdf-070b252f88b3","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7239c70d-39cc-4030-abdf-070b252f88b3"}
{"level":"info","ts":"2026-06-10T10:33:54.411Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"7239c70d-39cc-4030-abdf-070b252f88b3","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:54.418Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","name":"long-running-scs76","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-scs76\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"long-running-scs76"}}
{"level":"info","ts":"2026-06-10T10:33:54.418Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8"}
{"level":"error","ts":"2026-06-10T10:33:54.418Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"4581b26c-9337-4f13-a8b2-7d66f79d55c8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-scs76\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:33:54.421Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"c7531c01-dfae-4694-9ece-ad5b957c1642","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3"}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3"}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:54.424Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"2a700182-6342-4d50-a1b9-ac11c7be1da3","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:54.429Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"c7531c01-dfae-4694-9ece-ad5b957c1642","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:33:54.429Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"c7531c01-dfae-4694-9ece-ad5b957c1642","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e"}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e"}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:54.774Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"85f18097-c17d-40e4-86fb-27e128fc3f5e","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:54.822Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a"}
{"level":"info","ts":"2026-06-10T10:33:54.822Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a"}
{"level":"info","ts":"2026-06-10T10:33:54.822Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:54.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:54.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:54.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:54.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:54.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:54.823Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","podName":"long-running-scs76-small-group-worker-4pdq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.823Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","podName":"long-running-scs76-small-group-worker-4pdq5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:54.823Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:54.823Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51ee1fc-5cf2-414c-a3d6-3949beed045a","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668"}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668"}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"newHead":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"}}
{"level":"info","ts":"2026-06-10T10:33:55.796Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7df300b2-9a06-4ce3-9abc-a43284054668","name":"long-running-scs76","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:33:55.808Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5e457013-cca4-46a5-b459-72d6c9c0c74f","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:55.808Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ec3e07-7799-4928-aa91-e8f87462ec3b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:55.808Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ec3e07-7799-4928-aa91-e8f87462ec3b","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:55.808Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ec3e07-7799-4928-aa91-e8f87462ec3b","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:55.809Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ec3e07-7799-4928-aa91-e8f87462ec3b"}
{"level":"info","ts":"2026-06-10T10:33:55.809Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"64ec3e07-7799-4928-aa91-e8f87462ec3b","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:55.809Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"7ec0560f-b58b-4cec-a1aa-ad9847b41f9e","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:55.809Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"7ec0560f-b58b-4cec-a1aa-ad9847b41f9e","pod":"long-running-scs76-head-cfpwz","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-10T10:33:55.816Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5e457013-cca4-46a5-b459-72d6c9c0c74f","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:33:55.820Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"7ec0560f-b58b-4cec-a1aa-ad9847b41f9e","certificate":"ray-head-cert-long-running-scs76","dnsNames":["long-running-scs76-head-svc","localhost","long-running-scs76-head-svc.test-ns-nrv9s.svc","long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.20","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:33:55.825Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5e457013-cca4-46a5-b459-72d6c9c0c74f","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:33:55.825Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"5e457013-cca4-46a5-b459-72d6c9c0c74f","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:55.836Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"7ec0560f-b58b-4cec-a1aa-ad9847b41f9e","certificate":"ray-worker-cert-long-running-scs76","dnsNames":["long-running-scs76-worker-svc","localhost","long-running-scs76-small-group","long-running-scs76-worker-svc.test-ns-nrv9s.svc","long-running-scs76-worker-svc.test-ns-nrv9s.svc.cluster.local","*.long-running-scs76-worker-svc.test-ns-nrv9s.svc","*.long-running-scs76-worker-svc.test-ns-nrv9s.svc.cluster.local","*-worker-*.test-ns-nrv9s.svc","*-worker-*.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.20","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:33:55.836Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"7ec0560f-b58b-4cec-a1aa-ad9847b41f9e"}
{"level":"info","ts":"2026-06-10T10:33:55.893Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2"}
{"level":"info","ts":"2026-06-10T10:33:55.893Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2"}
{"level":"info","ts":"2026-06-10T10:33:55.893Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:55.894Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b9b1585c-d546-49b6-b87c-008114c2baa2","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:56.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"1faf2c22-5daa-4b1c-8920-09b288eebb5d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:56.336Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"1faf2c22-5daa-4b1c-8920-09b288eebb5d","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:56.336Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"1faf2c22-5daa-4b1c-8920-09b288eebb5d","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:56.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"1faf2c22-5daa-4b1c-8920-09b288eebb5d"}
{"level":"info","ts":"2026-06-10T10:33:56.336Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"1faf2c22-5daa-4b1c-8920-09b288eebb5d","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447"}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447"}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:33:57.808Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"b51f58e3-8a13-48bc-a0d6-21e008e2b447","seconds":300}
{"level":"info","ts":"2026-06-10T10:33:59.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bcfc01f6-6120-4e90-9baf-20ff18c61643","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:33:59.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bcfc01f6-6120-4e90-9baf-20ff18c61643","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:33:59.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bcfc01f6-6120-4e90-9baf-20ff18c61643","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:33:59.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bcfc01f6-6120-4e90-9baf-20ff18c61643"}
{"level":"info","ts":"2026-06-10T10:33:59.337Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"bcfc01f6-6120-4e90-9baf-20ff18c61643","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a"}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a"}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","podName":"long-running-scs76-head-cfpwz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:00.817Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"7cff107f-6c86-4e0e-b601-c322d0033f9a","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f"}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f"}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:01.825Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:01.826Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:01.826Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:01.826Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:34:01.826Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"e407945f-9f20-46b1-9658-01295c849e6f","name":"long-running-scs76","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:33:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:01.839Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"90c8e77b-c6d1-412d-979d-2030227dd335","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:01.839Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"da427b19-4570-4f8d-bf1a-51de5768437c","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:01.839Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9d7664da-4b4e-4b99-bf16-737e3be53fa9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:01.839Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9d7664da-4b4e-4b99-bf16-737e3be53fa9","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:01.839Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9d7664da-4b4e-4b99-bf16-737e3be53fa9","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:01.839Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"90c8e77b-c6d1-412d-979d-2030227dd335","pod":"long-running-scs76-head-cfpwz","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-10T10:34:01.839Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"90c8e77b-c6d1-412d-979d-2030227dd335","pod":"long-running-scs76-small-group-worker-4pdq5","ip":"10.132.0.44"}
{"level":"info","ts":"2026-06-10T10:34:01.839Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9d7664da-4b4e-4b99-bf16-737e3be53fa9"}
{"level":"info","ts":"2026-06-10T10:34:01.839Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"9d7664da-4b4e-4b99-bf16-737e3be53fa9","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:34:01.850Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"da427b19-4570-4f8d-bf1a-51de5768437c","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:34:01.851Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"90c8e77b-c6d1-412d-979d-2030227dd335","certificate":"ray-head-cert-long-running-scs76","dnsNames":["long-running-scs76-head-svc","localhost","long-running-scs76-head-svc.test-ns-nrv9s.svc","long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.20","10.132.0.44","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:34:01.860Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"da427b19-4570-4f8d-bf1a-51de5768437c","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:34:01.860Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"da427b19-4570-4f8d-bf1a-51de5768437c","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:01.867Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"90c8e77b-c6d1-412d-979d-2030227dd335","certificate":"ray-worker-cert-long-running-scs76","dnsNames":["long-running-scs76-worker-svc","localhost","long-running-scs76-small-group","long-running-scs76-worker-svc.test-ns-nrv9s.svc","long-running-scs76-worker-svc.test-ns-nrv9s.svc.cluster.local","*.long-running-scs76-worker-svc.test-ns-nrv9s.svc","*.long-running-scs76-worker-svc.test-ns-nrv9s.svc.cluster.local","*-worker-*.test-ns-nrv9s.svc","*-worker-*.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.20","10.132.0.44","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:34:01.867Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"90c8e77b-c6d1-412d-979d-2030227dd335"}
{"level":"info","ts":"2026-06-10T10:34:02.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a8b4c927-9968-4947-92a4-56ccd12f7936","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:02.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a8b4c927-9968-4947-92a4-56ccd12f7936","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:02.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a8b4c927-9968-4947-92a4-56ccd12f7936","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:02.339Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a8b4c927-9968-4947-92a4-56ccd12f7936"}
{"level":"info","ts":"2026-06-10T10:34:02.339Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"a8b4c927-9968-4947-92a4-56ccd12f7936","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302"}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302"}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:03.839Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:03.840Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79a3da5d-bc56-4012-b7f0-b8e17c407302","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96"}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96"}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:04.947Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"79ad1e91-8f8b-47c6-82d3-ab693b871e96","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:05.339Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ec64657a-dca9-4827-9707-33597e3a6160","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:05.339Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ec64657a-dca9-4827-9707-33597e3a6160","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:05.340Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ec64657a-dca9-4827-9707-33597e3a6160","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:05.340Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ec64657a-dca9-4827-9707-33597e3a6160"}
{"level":"info","ts":"2026-06-10T10:34:05.340Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"ec64657a-dca9-4827-9707-33597e3a6160","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f"}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f"}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:05.959Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:05.960Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:34:05.960Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"30a7ad66-9da4-474f-bc64-8e922e7c848f","name":"long-running-scs76","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:05.972Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f6673c9e-ff40-4265-99f7-33740908ff96","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:05.972Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f6673c9e-ff40-4265-99f7-33740908ff96","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:05.972Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f6673c9e-ff40-4265-99f7-33740908ff96","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:05.972Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f6673c9e-ff40-4265-99f7-33740908ff96"}
{"level":"info","ts":"2026-06-10T10:34:05.972Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f6673c9e-ff40-4265-99f7-33740908ff96","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:34:05.972Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"60c8fe57-7dea-482f-83ef-a4473fcd563a","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:05.973Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"60c8fe57-7dea-482f-83ef-a4473fcd563a","pod":"long-running-scs76-head-cfpwz","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-10T10:34:05.973Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"60c8fe57-7dea-482f-83ef-a4473fcd563a","pod":"long-running-scs76-small-group-worker-4pdq5","ip":"10.132.0.44"}
{"level":"info","ts":"2026-06-10T10:34:05.973Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"60c8fe57-7dea-482f-83ef-a4473fcd563a","rayCluster":"long-running-scs76","podIPs":["10.134.0.20","10.132.0.44"]}
{"level":"info","ts":"2026-06-10T10:34:05.973Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"4cf22252-600f-4cce-916c-1f6fafc76e0a","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:06.003Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"4cf22252-600f-4cce-916c-1f6fafc76e0a","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:34:06.013Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"4cf22252-600f-4cce-916c-1f6fafc76e0a","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:34:06.013Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"4cf22252-600f-4cce-916c-1f6fafc76e0a","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b"}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b"}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:07.973Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","podName":"long-running-scs76-head-cfpwz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:07.974Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:07.974Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"5b52d148-2cf5-46fb-93ea-28037f222b8b","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:08.341Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfb7022d-1e28-464d-9894-385d3144ee0b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:08.341Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfb7022d-1e28-464d-9894-385d3144ee0b","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:08.341Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfb7022d-1e28-464d-9894-385d3144ee0b","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:08.341Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfb7022d-1e28-464d-9894-385d3144ee0b"}
{"level":"info","ts":"2026-06-10T10:34:08.341Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfb7022d-1e28-464d-9894-385d3144ee0b","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:34:11.342Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f096b7ae-97ed-48ed-be7b-58bd58d753bb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:11.342Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f096b7ae-97ed-48ed-be7b-58bd58d753bb","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:11.342Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f096b7ae-97ed-48ed-be7b-58bd58d753bb","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:11.342Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f096b7ae-97ed-48ed-be7b-58bd58d753bb"}
{"level":"info","ts":"2026-06-10T10:34:11.342Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f096b7ae-97ed-48ed-be7b-58bd58d753bb","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624"}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624"}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","podName":"long-running-scs76-small-group-worker-4pdq5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","podName":"long-running-scs76-small-group-worker-4pdq5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:34:12.858Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"1e3a0edc-3f08-437d-a4d5-2bc17c2ca624","name":"long-running-scs76","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:12.870Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"e8b7d815-41e9-42a3-97de-f0910b936bf2","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:12.871Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"63817a38-5053-48d5-8dab-f3695de46c86","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:12.871Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f8c37722-24b9-485c-86e7-765a1b1e3bb2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:12.871Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"63817a38-5053-48d5-8dab-f3695de46c86","pod":"long-running-scs76-head-cfpwz","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-10T10:34:12.871Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f8c37722-24b9-485c-86e7-765a1b1e3bb2","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:12.871Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"63817a38-5053-48d5-8dab-f3695de46c86","pod":"long-running-scs76-small-group-worker-4pdq5","ip":"10.132.0.44"}
{"level":"info","ts":"2026-06-10T10:34:12.871Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f8c37722-24b9-485c-86e7-765a1b1e3bb2","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:12.871Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"63817a38-5053-48d5-8dab-f3695de46c86","rayCluster":"long-running-scs76","podIPs":["10.134.0.20","10.132.0.44"]}
{"level":"info","ts":"2026-06-10T10:34:12.871Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f8c37722-24b9-485c-86e7-765a1b1e3bb2"}
{"level":"info","ts":"2026-06-10T10:34:12.871Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"f8c37722-24b9-485c-86e7-765a1b1e3bb2","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:34:12.883Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"e8b7d815-41e9-42a3-97de-f0910b936bf2","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:34:12.894Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"e8b7d815-41e9-42a3-97de-f0910b936bf2","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:34:12.894Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"e8b7d815-41e9-42a3-97de-f0910b936bf2","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:14.343Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e05a1b3d-9341-4eb9-9596-893f1dc8bd2f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:14.343Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e05a1b3d-9341-4eb9-9596-893f1dc8bd2f","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:14.343Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e05a1b3d-9341-4eb9-9596-893f1dc8bd2f","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:14.344Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e05a1b3d-9341-4eb9-9596-893f1dc8bd2f"}
{"level":"info","ts":"2026-06-10T10:34:14.344Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"e05a1b3d-9341-4eb9-9596-893f1dc8bd2f","RayCluster":"long-running-scs76","State":""}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567"}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567"}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:14.871Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","podName":"long-running-scs76-small-group-worker-4pdq5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:34:14.872Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","podName":"long-running-scs76-small-group-worker-4pdq5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:33:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:34:14.872Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:14.872Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"0e62f093-0f03-40bc-8c5b-df710a49e567","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde"}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde"}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","1 head service found":"long-running-scs76-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","Found 1 head Pod":"long-running-scs76-head-cfpwz","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","head Pod":"long-running-scs76-head-cfpwz","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-scs76-head-cfpwz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","worker Pod":"long-running-scs76-small-group-worker-4pdq5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-scs76-small-group-worker-4pdq5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:34:16.987Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"ecaebadf-561a-4348-93a3-c366607e0dde","name":"long-running-scs76","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:16Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.20","serviceIP":"10.134.0.20","podName":"long-running-scs76-head-cfpwz","serviceName":"long-running-scs76-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:33:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:33:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:17.001Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6f4f3a64-3034-4da5-bc9d-d1d6e7ec949d","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.001Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"9535bcb2-4358-4946-9eac-8a63cbdcc0c2","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.001Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.001Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.001Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"9535bcb2-4358-4946-9eac-8a63cbdcc0c2","pod":"long-running-scs76-head-cfpwz","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-10T10:34:17.001Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"9535bcb2-4358-4946-9eac-8a63cbdcc0c2","pod":"long-running-scs76-small-group-worker-4pdq5","ip":"10.132.0.44"}
{"level":"info","ts":"2026-06-10T10:34:17.001Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72"}
{"level":"info","ts":"2026-06-10T10:34:17.002Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","head service name":"long-running-scs76-head-svc","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:17.002Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","head service URL":"long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:34:17.002Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72"}
{"level":"info","ts":"2026-06-10T10:34:17.002Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","command":["if","!","ray","job","status","--address","http://long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local:8265","long-running-ng82r",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local:8265","--no-wait","--submission-id","long-running-ng82r","--","python /home/ray/jobs/long_running.py",";","fi",";","ray","job","logs","--address","http://long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local:8265","--follow","long-running-ng82r"]}
{"level":"info","ts":"2026-06-10T10:34:17.001Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"9535bcb2-4358-4946-9eac-8a63cbdcc0c2","rayCluster":"long-running-scs76","podIPs":["10.134.0.20","10.132.0.44"]}
{"level":"info","ts":"2026-06-10T10:34:17.022Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-06-10T10:34:17.022Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","SubmissionMode":"K8sJobMode","RayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.022Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ng82r","rayClusterName":"long-running-scs76","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:33:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-ng82r","rayClusterName":"long-running-scs76","dashboardURL":"long-running-scs76-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:33:51Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:34:17.022Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"5a9a3cc0-19b2-4f4b-95cd-25f75f31ff72","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:34:17.026Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6f4f3a64-3034-4da5-bc9d-d1d6e7ec949d","name":"long-running-scs76-head"}
{"level":"info","ts":"2026-06-10T10:34:17.039Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d7a08089-71ff-4285-a6d1-a94d57b40c77","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.039Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d7a08089-71ff-4285-a6d1-a94d57b40c77","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.039Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d7a08089-71ff-4285-a6d1-a94d57b40c77","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.039Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d7a08089-71ff-4285-a6d1-a94d57b40c77"}
{"level":"error","ts":"2026-06-10T10:34:17.050Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d7a08089-71ff-4285-a6d1-a94d57b40c77","JobId":"long-running-ng82r","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.050Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d7a08089-71ff-4285-a6d1-a94d57b40c77"}
{"level":"error","ts":"2026-06-10T10:34:17.050Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"d7a08089-71ff-4285-a6d1-a94d57b40c77","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.056Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"afebfe58-ae3f-4dc9-a883-3f9d67e36d9b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.056Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"afebfe58-ae3f-4dc9-a883-3f9d67e36d9b","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.056Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"afebfe58-ae3f-4dc9-a883-3f9d67e36d9b","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.056Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"afebfe58-ae3f-4dc9-a883-3f9d67e36d9b"}
{"level":"info","ts":"2026-06-10T10:34:17.099Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6f4f3a64-3034-4da5-bc9d-d1d6e7ec949d","name":"long-running-scs76-workers"}
{"level":"info","ts":"2026-06-10T10:34:17.099Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6f4f3a64-3034-4da5-bc9d-d1d6e7ec949d","cluster":"long-running-scs76"}
{"level":"error","ts":"2026-06-10T10:34:17.103Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"afebfe58-ae3f-4dc9-a883-3f9d67e36d9b","JobId":"long-running-ng82r","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.103Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"afebfe58-ae3f-4dc9-a883-3f9d67e36d9b"}
{"level":"error","ts":"2026-06-10T10:34:17.103Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"afebfe58-ae3f-4dc9-a883-3f9d67e36d9b","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.104Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"784856bb-dd0d-428a-b542-4db8a04e1d47","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.104Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"784856bb-dd0d-428a-b542-4db8a04e1d47","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.104Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"784856bb-dd0d-428a-b542-4db8a04e1d47","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.104Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"784856bb-dd0d-428a-b542-4db8a04e1d47"}
{"level":"error","ts":"2026-06-10T10:34:17.109Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"784856bb-dd0d-428a-b542-4db8a04e1d47","JobId":"long-running-ng82r","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.109Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"784856bb-dd0d-428a-b542-4db8a04e1d47"}
{"level":"error","ts":"2026-06-10T10:34:17.109Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"784856bb-dd0d-428a-b542-4db8a04e1d47","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.114Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfddbde6-c1d9-444c-b202-384a2fc98a74","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.114Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfddbde6-c1d9-444c-b202-384a2fc98a74","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.114Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfddbde6-c1d9-444c-b202-384a2fc98a74","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.114Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfddbde6-c1d9-444c-b202-384a2fc98a74"}
{"level":"error","ts":"2026-06-10T10:34:17.118Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfddbde6-c1d9-444c-b202-384a2fc98a74","JobId":"long-running-ng82r","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.118Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfddbde6-c1d9-444c-b202-384a2fc98a74"}
{"level":"error","ts":"2026-06-10T10:34:17.118Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"dfddbde6-c1d9-444c-b202-384a2fc98a74","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.159Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"cfe69570-09fe-4fc1-b7f5-fd3752c1357d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.159Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"cfe69570-09fe-4fc1-b7f5-fd3752c1357d","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.160Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"cfe69570-09fe-4fc1-b7f5-fd3752c1357d","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.160Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"cfe69570-09fe-4fc1-b7f5-fd3752c1357d"}
{"level":"error","ts":"2026-06-10T10:34:17.165Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"cfe69570-09fe-4fc1-b7f5-fd3752c1357d","JobId":"long-running-ng82r","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.165Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"cfe69570-09fe-4fc1-b7f5-fd3752c1357d"}
{"level":"error","ts":"2026-06-10T10:34:17.165Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"cfe69570-09fe-4fc1-b7f5-fd3752c1357d","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.245Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"44ba133c-9e26-400a-ac22-72b515306855","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"44ba133c-9e26-400a-ac22-72b515306855","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.245Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"44ba133c-9e26-400a-ac22-72b515306855","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.245Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"44ba133c-9e26-400a-ac22-72b515306855"}
{"level":"error","ts":"2026-06-10T10:34:17.251Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"44ba133c-9e26-400a-ac22-72b515306855","JobId":"long-running-ng82r","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.251Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"44ba133c-9e26-400a-ac22-72b515306855"}
{"level":"error","ts":"2026-06-10T10:34:17.251Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"44ba133c-9e26-400a-ac22-72b515306855","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.412Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"2a211e9e-fec9-47da-8eb8-289e1ba4269d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.412Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"2a211e9e-fec9-47da-8eb8-289e1ba4269d","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.412Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"2a211e9e-fec9-47da-8eb8-289e1ba4269d","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.412Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"2a211e9e-fec9-47da-8eb8-289e1ba4269d"}
{"level":"error","ts":"2026-06-10T10:34:17.418Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"2a211e9e-fec9-47da-8eb8-289e1ba4269d","JobId":"long-running-ng82r","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.418Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"2a211e9e-fec9-47da-8eb8-289e1ba4269d"}
{"level":"error","ts":"2026-06-10T10:34:17.418Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"2a211e9e-fec9-47da-8eb8-289e1ba4269d","error":"Job long-running-ng82r does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.488Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"80966237-ad83-411d-8aff-700bb2c9cb39","DeletionTimestamp":"2026-06-10 10:34:17 +0000 UTC"}
{"level":"info","ts":"2026-06-10T10:34:17.488Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"80966237-ad83-411d-8aff-700bb2c9cb39","rayJob":"long-running-ng82r"}
{"level":"error","ts":"2026-06-10T10:34:17.494Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"80966237-ad83-411d-8aff-700bb2c9cb39","error":"invalid character 'J' looking for beginning of value","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.494Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"80966237-ad83-411d-8aff-700bb2c9cb39","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:34:17.504Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"39beedd0-d51e-4b01-96a3-342055472c55","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.504Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"39beedd0-d51e-4b01-96a3-342055472c55","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:34:17.512Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"39beedd0-d51e-4b01-96a3-342055472c55"}
{"level":"info","ts":"2026-06-10T10:34:17.512Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"39beedd0-d51e-4b01-96a3-342055472c55","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:34:17.512Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"39beedd0-d51e-4b01-96a3-342055472c55","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kt66w","rayClusterName":"counter-f2zjz","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:34:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:34:17.512Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"39beedd0-d51e-4b01-96a3-342055472c55","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"25c331e1-9ced-4675-8923-70294c149c20","namespacedName":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"25c331e1-9ced-4675-8923-70294c149c20","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"25c331e1-9ced-4675-8923-70294c149c20","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"fbb74dd4-582a-4ad0-a14b-d02886ed9a2b"}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6c72c473-0ffc-4cbb-9e2b-355f1c635e29","rayCluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"25c331e1-9ced-4675-8923-70294c149c20","namespace":"test-ns-nrv9s","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"reconcileID":"33752613-cae0-4fc3-bd66-e314e1b0ce3d"}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6c72c473-0ffc-4cbb-9e2b-355f1c635e29","pod":"long-running-scs76-head-cfpwz","ip":"10.134.0.20"}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6c72c473-0ffc-4cbb-9e2b-355f1c635e29","pod":"long-running-scs76-small-group-worker-4pdq5","ip":"10.132.0.44"}
{"level":"info","ts":"2026-06-10T10:34:17.523Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6c72c473-0ffc-4cbb-9e2b-355f1c635e29","rayCluster":"long-running-scs76","podIPs":["10.134.0.20","10.132.0.44"]}
{"level":"info","ts":"2026-06-10T10:34:17.528Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b7799e74-1985-4d4f-bea7-2a9b90327d49"}
{"level":"info","ts":"2026-06-10T10:34:17.528Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4963e051-f8bf-4858-b475-e7961638db07","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.528Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4963e051-f8bf-4858-b475-e7961638db07","oldStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:34:17.528Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4963e051-f8bf-4858-b475-e7961638db07","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kt66w","rayClusterName":"counter-f2zjz","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:34:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kt66w","rayClusterName":"counter-f2zjz","jobDeploymentStatus":"Suspending","startTime":"2026-06-10T10:34:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:34:17.529Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4963e051-f8bf-4858-b475-e7961638db07","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-06-10T10:34:17.535Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"25c331e1-9ced-4675-8923-70294c149c20","name":"kuberay-gateway-access","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:17.540Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"139a5c91-008c-4411-91bb-2e80eba53b67","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.540Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"139a5c91-008c-4411-91bb-2e80eba53b67","RayCluster":{"name":"counter-f2zjz","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.540Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"139a5c91-008c-4411-91bb-2e80eba53b67","isClusterDeleted":true}
{"level":"info","ts":"2026-06-10T10:34:17.540Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"139a5c91-008c-4411-91bb-2e80eba53b67","Kubernetes Job":""}
{"level":"info","ts":"2026-06-10T10:34:17.540Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"139a5c91-008c-4411-91bb-2e80eba53b67","isJobDeleted":true}
{"level":"info","ts":"2026-06-10T10:34:17.540Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"139a5c91-008c-4411-91bb-2e80eba53b67","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-kt66w","rayClusterName":"counter-f2zjz","jobDeploymentStatus":"Suspending","startTime":"2026-06-10T10:34:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-10T10:34:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:34:17.540Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"139a5c91-008c-4411-91bb-2e80eba53b67","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-06-10T10:34:17.541Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"25c331e1-9ced-4675-8923-70294c149c20","name":"test-ns-nrv9s-long-running-scs76","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:34:17.547Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"25c331e1-9ced-4675-8923-70294c149c20","configMap":"kube-rbac-proxy-config-long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.551Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7a62cba8-2f67-41cb-86bc-b7b11a3b66e4","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:17.552Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"25c331e1-9ced-4675-8923-70294c149c20","serviceAccount":"long-running-scs76-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:34:17.599Z","logger":"raycluster-resource","msg":"default","name":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.599Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-scs76","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:17.610Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"25c331e1-9ced-4675-8923-70294c149c20","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.610Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1503d031-fae3-48d1-8a2f-ab464e5cc631","namespacedName":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.610Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1503d031-fae3-48d1-8a2f-ab464e5cc631","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.610Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1503d031-fae3-48d1-8a2f-ab464e5cc631","cluster":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.610Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1503d031-fae3-48d1-8a2f-ab464e5cc631","namespace":"test-ns-nrv9s","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:34:17.610Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1ebdf6a2-286b-43c5-85c1-6fd5f74f7280","namespace":"test-ns-nrv9s","clusterName":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.610Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1503d031-fae3-48d1-8a2f-ab464e5cc631","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:34:17.610Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"bed93636-1e17-4013-8cd9-46110346bbe6"}
{"level":"info","ts":"2026-06-10T10:34:17.610Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"b9aca5d6-709e-494a-8977-c41e439c84dd"}
{"level":"error","ts":"2026-06-10T10:34:17.613Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1503d031-fae3-48d1-8a2f-ab464e5cc631","error":"rayclusters.ray.io \"long-running-scs76\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.613Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1503d031-fae3-48d1-8a2f-ab464e5cc631"}
{"level":"error","ts":"2026-06-10T10:34:17.613Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1503d031-fae3-48d1-8a2f-ab464e5cc631","error":"rayclusters.ray.io \"long-running-scs76\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:17.613Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6307eb57-c2a2-49ed-987a-23e8a6fa1111","namespacedName":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.613Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6307eb57-c2a2-49ed-987a-23e8a6fa1111"}
{"level":"info","ts":"2026-06-10T10:34:17.613Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6307eb57-c2a2-49ed-987a-23e8a6fa1111","namespace":"test-ns-nrv9s","authClusters":0}
{"level":"info","ts":"2026-06-10T10:34:17.613Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"6307eb57-c2a2-49ed-987a-23e8a6fa1111","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:17.619Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"36ba2794-8769-4b5a-b727-7ab321b19400","namespacedName":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.619Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"36ba2794-8769-4b5a-b727-7ab321b19400"}
{"level":"info","ts":"2026-06-10T10:34:17.619Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"36ba2794-8769-4b5a-b727-7ab321b19400","namespace":"test-ns-nrv9s","authClusters":0}
{"level":"info","ts":"2026-06-10T10:34:17.619Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"36ba2794-8769-4b5a-b727-7ab321b19400","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:17.620Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"20e844fa-2823-49fc-9257-4a2c43dcf9ff"}
{"level":"info","ts":"2026-06-10T10:34:17.621Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"71a24449-17f7-45c7-88f5-8a1fb4444c8c","namespacedName":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:17.621Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"71a24449-17f7-45c7-88f5-8a1fb4444c8c"}
{"level":"info","ts":"2026-06-10T10:34:17.621Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"71a24449-17f7-45c7-88f5-8a1fb4444c8c","namespace":"test-ns-nrv9s","authClusters":0}
{"level":"info","ts":"2026-06-10T10:34:17.621Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"71a24449-17f7-45c7-88f5-8a1fb4444c8c","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:17.628Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1bd929b0-a3db-47fe-b5ec-1bf850892578"}
{"level":"info","ts":"2026-06-10T10:34:17.699Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"1ebdf6a2-286b-43c5-85c1-6fd5f74f7280","namespace":"test-ns-nrv9s","clusterName":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:17.738Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-nrv9s"},"reconcileID":"3227b270-5a94-4330-9408-102107b1362a"}
{"level":"info","ts":"2026-06-10T10:34:18.521Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6acc2ce6-cb00-4ed1-a6e6-de26559874e6","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:18.521Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6acc2ce6-cb00-4ed1-a6e6-de26559874e6"}
{"level":"info","ts":"2026-06-10T10:34:18.521Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6acc2ce6-cb00-4ed1-a6e6-de26559874e6","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-06-10T10:34:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-10T10:34:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:34:18.521Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6acc2ce6-cb00-4ed1-a6e6-de26559874e6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-06-10T10:34:18.531Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"49ebe941-56c4-4c3d-927e-fb532cfbd543","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:18.531Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"49ebe941-56c4-4c3d-927e-fb532cfbd543"}
{"level":"info","ts":"2026-06-10T10:34:18.531Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"49ebe941-56c4-4c3d-927e-fb532cfbd543","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:34:18.531Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"49ebe941-56c4-4c3d-927e-fb532cfbd543","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-06-10T10:34:17Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:34:18.531Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"49ebe941-56c4-4c3d-927e-fb532cfbd543","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:34:18.542Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5f279721-39be-4c58-b643-e6257ec55b59","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:18.542Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5f279721-39be-4c58-b643-e6257ec55b59","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.542Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5f279721-39be-4c58-b643-e6257ec55b59","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.542Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5f279721-39be-4c58-b643-e6257ec55b59","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.546Z","logger":"raycluster-resource","msg":"default","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.546Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-pc8bg","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:18.553Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5f279721-39be-4c58-b643-e6257ec55b59","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5f279721-39be-4c58-b643-e6257ec55b59"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5f279721-39be-4c58-b643-e6257ec55b59","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6112365b-3171-4d82-8dbd-7829af914a03","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4e778d00-0f8d-41b5-99a4-ee77d7bc4c86","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4e778d00-0f8d-41b5-99a4-ee77d7bc4c86","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4e778d00-0f8d-41b5-99a4-ee77d7bc4c86","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","namespacedName":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","mode":"IntegratedOAuth","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4e778d00-0f8d-41b5-99a4-ee77d7bc4c86"}
{"level":"info","ts":"2026-06-10T10:34:18.554Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4e778d00-0f8d-41b5-99a4-ee77d7bc4c86","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:18.559Z","logger":"raycluster-resource","msg":"default","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.559Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-pc8bg","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:18.564Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","name":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:18.564Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","error":"head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"error","ts":"2026-06-10T10:34:18.564Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.564Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"info","ts":"2026-06-10T10:34:18.564Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.564Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.564Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.564Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:34:18.564Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","name":"counter-pc8bg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:18.599Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.599Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.599Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6112365b-3171-4d82-8dbd-7829af914a03","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.599Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"849a96c9-0fcb-461c-9162-ef78a1240e68","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.600Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"d472646b-0c28-4021-a1df-b41867f0e9c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:18.600Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"d472646b-0c28-4021-a1df-b41867f0e9c1","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.600Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"d472646b-0c28-4021-a1df-b41867f0e9c1","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"d472646b-0c28-4021-a1df-b41867f0e9c1"}
{"level":"info","ts":"2026-06-10T10:34:18.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"d472646b-0c28-4021-a1df-b41867f0e9c1","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:18.605Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","name":"counter-pc8bg-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:34:18.607Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","name":"counter-pc8bg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-pc8bg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"counter-pc8bg"}}
{"level":"info","ts":"2026-06-10T10:34:18.607Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9"}
{"level":"error","ts":"2026-06-10T10:34:18.607Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e1761203-02fd-4bc9-928b-c90ef89fd2b9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.607Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7"}
{"level":"info","ts":"2026-06-10T10:34:18.607Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7"}
{"level":"info","ts":"2026-06-10T10:34:18.607Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:18.607Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","error":"head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"error","ts":"2026-06-10T10:34:18.607Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.607Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"info","ts":"2026-06-10T10:34:18.608Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.608Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.608Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.608Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:34:18.608Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","name":"counter-pc8bg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:18.608Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221"}
{"level":"info","ts":"2026-06-10T10:34:18.608Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","rayCluster":"counter-pc8bg","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:34:18.608Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.612Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"849a96c9-0fcb-461c-9162-ef78a1240e68","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:18.612Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","name":"kuberay-gateway-access","namespace":"test-ns-nrv9s","operation":"created"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7"}
{"level":"error","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"469af10c-fff7-43e8-b7a3-2a530a9352d7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","error":"head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"error","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","name":"counter-pc8bg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"48149f2d-40c5-45ee-abd5-1100e577ce56","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"48149f2d-40c5-45ee-abd5-1100e577ce56","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"48149f2d-40c5-45ee-abd5-1100e577ce56","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"48149f2d-40c5-45ee-abd5-1100e577ce56"}
{"level":"info","ts":"2026-06-10T10:34:18.618Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"48149f2d-40c5-45ee-abd5-1100e577ce56","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:18.621Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.699Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221"}
{"level":"info","ts":"2026-06-10T10:34:18.699Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","name":"test-ns-nrv9s-counter-pc8bg","operation":"created"}
{"level":"info","ts":"2026-06-10T10:34:18.699Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","name":"counter-pc8bg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-pc8bg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"counter-pc8bg"}}
{"level":"info","ts":"2026-06-10T10:34:18.699Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","rayCluster":"counter-pc8bg","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:34:18.699Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880"}
{"level":"error","ts":"2026-06-10T10:34:18.699Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"4f63244f-17fb-4b45-b9d4-d07f323be880","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859"}
{"level":"info","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859"}
{"level":"info","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859","error":"head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"error","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"info","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859"}
{"level":"error","ts":"2026-06-10T10:34:18.700Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"c4251c4c-1d57-4c30-9db4-6fdba0d3e859","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.702Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"849a96c9-0fcb-461c-9162-ef78a1240e68","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:18.702Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"849a96c9-0fcb-461c-9162-ef78a1240e68","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.702Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6f317908-3374-4a00-9488-7399595a0d30","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.703Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","name":"kube-rbac-proxy-config-counter-pc8bg","operation":"created"}
{"level":"info","ts":"2026-06-10T10:34:18.709Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221"}
{"level":"info","ts":"2026-06-10T10:34:18.710Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","rayCluster":"counter-pc8bg","podIPs":[]}
{"level":"error","ts":"2026-06-10T10:34:18.710Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-pc8bg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.710Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221"}
{"level":"error","ts":"2026-06-10T10:34:18.710Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"49d393d2-3506-435c-83ef-999cff4ad221","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-pc8bg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.710Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"647239b8-ae59-4ef3-b324-4ca2c028fd1c","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.710Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"647239b8-ae59-4ef3-b324-4ca2c028fd1c"}
{"level":"error","ts":"2026-06-10T10:34:18.710Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-pc8bg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.710Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7"}
{"level":"error","ts":"2026-06-10T10:34:18.710Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ea56d073-40a3-4668-8653-389242ac46b7","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-pc8bg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.710Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"4889cafa-e950-425e-a3f2-6785a7467d69","namespacedName":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.710Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"4889cafa-e950-425e-a3f2-6785a7467d69","mode":"IntegratedOAuth","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.710Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"4889cafa-e950-425e-a3f2-6785a7467d69","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.713Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6f317908-3374-4a00-9488-7399595a0d30","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:18.715Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"471ba14d-a05f-4142-a6d1-cafe9a9c7b99","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.715Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"471ba14d-a05f-4142-a6d1-cafe9a9c7b99"}
{"level":"info","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08"}
{"level":"info","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08"}
{"level":"info","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08","error":"head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"error","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"info","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08"}
{"level":"error","ts":"2026-06-10T10:34:18.720Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3272e781-efce-4f52-b957-08d6cb383c08","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.799Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"4889cafa-e950-425e-a3f2-6785a7467d69","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.800Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"b20a03b9-55de-4f16-8465-7679af5f628d","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.800Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ddf7779e-4f7c-4d4d-9e7a-c2c12bd7b48f","namespacedName":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ddf7779e-4f7c-4d4d-9e7a-c2c12bd7b48f","mode":"IntegratedOAuth","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ddf7779e-4f7c-4d4d-9e7a-c2c12bd7b48f","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e692ac8c-5894-446a-863f-c5c2ddd89b2a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:18.801Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"b20a03b9-55de-4f16-8465-7679af5f628d"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e692ac8c-5894-446a-863f-c5c2ddd89b2a","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e692ac8c-5894-446a-863f-c5c2ddd89b2a","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:18.807Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6f317908-3374-4a00-9488-7399595a0d30","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6f317908-3374-4a00-9488-7399595a0d30","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9","error":"head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"error","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"0b6d82f7-a28e-4b06-842f-c13dc41ca32f","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e692ac8c-5894-446a-863f-c5c2ddd89b2a"}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e692ac8c-5894-446a-863f-c5c2ddd89b2a","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9"}
{"level":"error","ts":"2026-06-10T10:34:18.899Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e6f46f15-3f37-4b87-be1b-174bb4e31ba9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:18.908Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ddf7779e-4f7c-4d4d-9e7a-c2c12bd7b48f","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:18.910Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"0b6d82f7-a28e-4b06-842f-c13dc41ca32f","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:18.921Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"0b6d82f7-a28e-4b06-842f-c13dc41ca32f","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:18.921Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"0b6d82f7-a28e-4b06-842f-c13dc41ca32f","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:19.060Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf"}
{"level":"info","ts":"2026-06-10T10:34:19.060Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf"}
{"level":"info","ts":"2026-06-10T10:34:19.060Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:19.060Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf","error":"head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"error","ts":"2026-06-10T10:34:19.060Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:19.061Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"info","ts":"2026-06-10T10:34:19.061Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:19.061Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:19.061Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:19.061Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf"}
{"level":"error","ts":"2026-06-10T10:34:19.061Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"3395be99-6318-4676-a44c-13c5d25bffcf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:19.381Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80"}
{"level":"info","ts":"2026-06-10T10:34:19.381Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80"}
{"level":"info","ts":"2026-06-10T10:34:19.381Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:19.382Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80","error":"head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"error","ts":"2026-06-10T10:34:19.382Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:19.382Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found"}
{"level":"info","ts":"2026-06-10T10:34:19.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:19.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:19.382Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:19.382Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80"}
{"level":"error","ts":"2026-06-10T10:34:19.382Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"f35d7609-d5f0-46bf-896b-d5d49f53ec80","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-pc8bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf"}
{"level":"info","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf"}
{"level":"info","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf","error":"worker certificate secret ray-worker-secret-counter-pc8bg not found"}
{"level":"error","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-counter-pc8bg not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-counter-pc8bg not found"}
{"level":"info","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf","filter labels":["test-ns-nrv9s",{"ray.io/cluster":"counter-pc8bg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf"}
{"level":"error","ts":"2026-06-10T10:34:20.022Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"63f4dd0e-fbf6-45ab-80ff-e609e25a8adf","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-counter-pc8bg not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:20.529Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"548c1c6c-e983-4103-a1af-e447f0bade2d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:20.529Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"548c1c6c-e983-4103-a1af-e447f0bade2d","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:20.529Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"548c1c6c-e983-4103-a1af-e447f0bade2d","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:20.529Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"548c1c6c-e983-4103-a1af-e447f0bade2d"}
{"level":"info","ts":"2026-06-10T10:34:20.529Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"548c1c6c-e983-4103-a1af-e447f0bade2d","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:21.303Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0"}
{"level":"info","ts":"2026-06-10T10:34:21.303Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0"}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0"}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0"}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","mode":"IntegratedOAuth","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","cluster":"counter-pc8bg","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","cluster":"counter-pc8bg","authType":"IntegratedOAuth","serviceAccount":"counter-pc8bg-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-pc8bg","ray.io/group":"headgroup","ray.io/identifier":"counter-pc8bg-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:34:21.304Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","enabled":true}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","name":"counter-pc8bg-head-nwnwp"}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0"}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","nodeType":"worker","rayStartParams":{"address":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","rayStartCmd":"ray start  --address=counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:34:21.318Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","enabled":true}
{"level":"info","ts":"2026-06-10T10:34:21.331Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","name":"counter-pc8bg-small-group-worker-w2rl6"}
{"level":"info","ts":"2026-06-10T10:34:21.331Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.332Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.332Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-pc8bg-head-svc"},"newHead":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"}}
{"level":"info","ts":"2026-06-10T10:34:21.332Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"03f28821-c796-4a85-8caf-f5ed865ba3b0","name":"counter-pc8bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"eda37e58-a55e-4713-80fd-7c7732c9e041","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-pc8bg-head-svc"},"newHead":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"}}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"80850722-beee-4901-a27a-4ae94a65ecf7","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.345Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","name":"counter-pc8bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:21.399Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5b3c30eb-68f2-4d75-bbb1-f556be3e7af5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:21.399Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5b3c30eb-68f2-4d75-bbb1-f556be3e7af5","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.399Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5b3c30eb-68f2-4d75-bbb1-f556be3e7af5","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:21.399Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"eda37e58-a55e-4713-80fd-7c7732c9e041","rayCluster":"counter-pc8bg","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:34:21.400Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5b3c30eb-68f2-4d75-bbb1-f556be3e7af5"}
{"level":"info","ts":"2026-06-10T10:34:21.400Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"5b3c30eb-68f2-4d75-bbb1-f556be3e7af5","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:21.499Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","name":"counter-pc8bg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-pc8bg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"counter-pc8bg"}}
{"level":"info","ts":"2026-06-10T10:34:21.499Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44"}
{"level":"error","ts":"2026-06-10T10:34:21.499Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"0c3d4737-d40a-4ecc-af3c-364ca99f8a44","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-pc8bg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:21.500Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75"}
{"level":"info","ts":"2026-06-10T10:34:21.500Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75"}
{"level":"info","ts":"2026-06-10T10:34:21.504Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"80850722-beee-4901-a27a-4ae94a65ecf7","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"80850722-beee-4901-a27a-4ae94a65ecf7","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"80850722-beee-4901-a27a-4ae94a65ecf7","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:21.599Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.600Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.600Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.600Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:34:21.600Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"472f0cad-972e-48ed-bd95-8553b9754c75","name":"counter-pc8bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"df89d998-1b9e-4a8f-9efc-e8f1c6e8c892","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"00ca9a0b-e121-49b0-b899-3464632c1ebf","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","podName":"counter-pc8bg-small-group-worker-w2rl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","podName":"counter-pc8bg-small-group-worker-w2rl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"df89d998-1b9e-4a8f-9efc-e8f1c6e8c892","rayCluster":"counter-pc8bg","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:34:21.612Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:34:21.613Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"696bc751-7452-4cd1-935a-8e7409d23103","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:21.613Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"696bc751-7452-4cd1-935a-8e7409d23103","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.613Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"696bc751-7452-4cd1-935a-8e7409d23103","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:21.613Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","name":"counter-pc8bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:21.613Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"696bc751-7452-4cd1-935a-8e7409d23103"}
{"level":"info","ts":"2026-06-10T10:34:21.613Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"696bc751-7452-4cd1-935a-8e7409d23103","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:21.704Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"00ca9a0b-e121-49b0-b899-3464632c1ebf","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:21.706Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","name":"counter-pc8bg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-pc8bg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-nrv9s","name":"counter-pc8bg"}}
{"level":"info","ts":"2026-06-10T10:34:21.706Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139"}
{"level":"error","ts":"2026-06-10T10:34:21.706Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"a6f397b8-8c20-49f3-b7ab-adaa7efd2139","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-pc8bg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:21.711Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8"}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8"}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:21.712Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b43170c1-abf6-415a-afeb-61eecb83ead8","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:21.714Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"00ca9a0b-e121-49b0-b899-3464632c1ebf","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:21.714Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"00ca9a0b-e121-49b0-b899-3464632c1ebf","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.722Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"55a4edc0-6aac-46d8-9a38-e90096c20da8","namespace":"test-ns-nrv9s","clusterName":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:21.722Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-scs76","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"long-running-scs76","reconcileID":"55a4edc0-6aac-46d8-9a38-e90096c20da8","namespace":"test-ns-nrv9s","clusterName":"long-running-scs76"}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569"}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569"}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.772Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.773Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:21.773Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"48dbfbf2-6762-4d3c-ae4f-2f4f87f69569","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7"}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7"}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.788Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.789Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.789Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:21.789Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"939f7c80-f181-4adf-bf3c-e77a7aa80bb7","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119"}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119"}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"newHead":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"}}
{"level":"info","ts":"2026-06-10T10:34:21.904Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"e5f7eb0a-13e5-4cd0-90e7-cf2cd7d82119","name":"counter-pc8bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:21.917Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ec84c483-e24c-4633-a257-da2904a58963","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.917Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ced54a35-b820-4402-90d1-9c246a4376fd","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.917Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ced54a35-b820-4402-90d1-9c246a4376fd","pod":"counter-pc8bg-head-nwnwp","ip":"10.134.0.22"}
{"level":"info","ts":"2026-06-10T10:34:21.917Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"c3a64a68-90a1-4760-8d1b-56f85ceb28a5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:21.917Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"c3a64a68-90a1-4760-8d1b-56f85ceb28a5","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:21.917Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"c3a64a68-90a1-4760-8d1b-56f85ceb28a5","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:21.917Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"c3a64a68-90a1-4760-8d1b-56f85ceb28a5"}
{"level":"info","ts":"2026-06-10T10:34:21.917Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"c3a64a68-90a1-4760-8d1b-56f85ceb28a5","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:21.928Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ec84c483-e24c-4633-a257-da2904a58963","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:21.929Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ced54a35-b820-4402-90d1-9c246a4376fd","certificate":"ray-head-cert-counter-pc8bg","dnsNames":["counter-pc8bg-head-svc","localhost","counter-pc8bg-head-svc.test-ns-nrv9s.svc","counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.22","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:34:21.941Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ced54a35-b820-4402-90d1-9c246a4376fd","certificate":"ray-worker-cert-counter-pc8bg","dnsNames":["counter-pc8bg-worker-svc","localhost","counter-pc8bg-small-group","counter-pc8bg-worker-svc.test-ns-nrv9s.svc","counter-pc8bg-worker-svc.test-ns-nrv9s.svc.cluster.local","*.counter-pc8bg-worker-svc.test-ns-nrv9s.svc","*.counter-pc8bg-worker-svc.test-ns-nrv9s.svc.cluster.local","*-worker-*.test-ns-nrv9s.svc","*-worker-*.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.22","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:34:21.941Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ced54a35-b820-4402-90d1-9c246a4376fd","rayCluster":"counter-pc8bg","podIPs":["10.134.0.22"]}
{"level":"info","ts":"2026-06-10T10:34:21.943Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ec84c483-e24c-4633-a257-da2904a58963","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:21.944Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"ec84c483-e24c-4633-a257-da2904a58963","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15"}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15"}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:22.060Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2a8eba90-4b78-4e40-8a41-2dd832c85a15","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:23.530Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7f28d05f-1528-4b6e-8024-80a88415c0cf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:23.530Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7f28d05f-1528-4b6e-8024-80a88415c0cf","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:23.530Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7f28d05f-1528-4b6e-8024-80a88415c0cf","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:23.530Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7f28d05f-1528-4b6e-8024-80a88415c0cf"}
{"level":"info","ts":"2026-06-10T10:34:23.530Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7f28d05f-1528-4b6e-8024-80a88415c0cf","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:23.917Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889"}
{"level":"info","ts":"2026-06-10T10:34:23.917Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889"}
{"level":"info","ts":"2026-06-10T10:34:23.917Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:23.918Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"aa437db1-3af2-4603-8ccb-46bcadab3889","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:26.531Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"86046698-0085-4e12-aa31-86f012e185aa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:26.531Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"86046698-0085-4e12-aa31-86f012e185aa","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:26.531Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"86046698-0085-4e12-aa31-86f012e185aa","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:26.531Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"86046698-0085-4e12-aa31-86f012e185aa"}
{"level":"info","ts":"2026-06-10T10:34:26.531Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"86046698-0085-4e12-aa31-86f012e185aa","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc"}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc"}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","podName":"counter-pc8bg-head-nwnwp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:26.926Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"9204759e-d4cc-4137-90c5-9a494cdaa5dc","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335"}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335"}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:27.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:27.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","podName":"counter-pc8bg-head-nwnwp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:27.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","podName":"counter-pc8bg-head-nwnwp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:27.932Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:34:27.932Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"d1346be6-603b-4b36-be1e-50b5f2ac0335","name":"counter-pc8bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:27.943Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"43c9ee7c-7880-453e-a458-1fc47da79e73","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:27.943Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"f5dfd122-a74c-46fc-a663-b98234f4b8f8","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:27.943Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"1a399497-bb6f-430e-9f34-c7d3fec80904","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:27.943Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"1a399497-bb6f-430e-9f34-c7d3fec80904","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:27.943Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"1a399497-bb6f-430e-9f34-c7d3fec80904","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:27.943Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"43c9ee7c-7880-453e-a458-1fc47da79e73","pod":"counter-pc8bg-head-nwnwp","ip":"10.134.0.22"}
{"level":"info","ts":"2026-06-10T10:34:27.943Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"43c9ee7c-7880-453e-a458-1fc47da79e73","pod":"counter-pc8bg-small-group-worker-w2rl6","ip":"10.132.0.45"}
{"level":"info","ts":"2026-06-10T10:34:27.943Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"1a399497-bb6f-430e-9f34-c7d3fec80904"}
{"level":"info","ts":"2026-06-10T10:34:27.943Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"1a399497-bb6f-430e-9f34-c7d3fec80904","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:27.955Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"43c9ee7c-7880-453e-a458-1fc47da79e73","certificate":"ray-head-cert-counter-pc8bg","dnsNames":["counter-pc8bg-head-svc","localhost","counter-pc8bg-head-svc.test-ns-nrv9s.svc","counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.22","10.132.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:34:27.955Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"f5dfd122-a74c-46fc-a663-b98234f4b8f8","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:27.964Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"f5dfd122-a74c-46fc-a663-b98234f4b8f8","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:27.964Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"f5dfd122-a74c-46fc-a663-b98234f4b8f8","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:27.968Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"43c9ee7c-7880-453e-a458-1fc47da79e73","certificate":"ray-worker-cert-counter-pc8bg","dnsNames":["counter-pc8bg-worker-svc","localhost","counter-pc8bg-small-group","counter-pc8bg-worker-svc.test-ns-nrv9s.svc","counter-pc8bg-worker-svc.test-ns-nrv9s.svc.cluster.local","*.counter-pc8bg-worker-svc.test-ns-nrv9s.svc","*.counter-pc8bg-worker-svc.test-ns-nrv9s.svc.cluster.local","*-worker-*.test-ns-nrv9s.svc","*-worker-*.test-ns-nrv9s.svc.cluster.local"],"ipAddresses":["10.134.0.22","10.132.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:34:27.969Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"43c9ee7c-7880-453e-a458-1fc47da79e73","rayCluster":"counter-pc8bg","podIPs":["10.134.0.22","10.132.0.45"]}
{"level":"info","ts":"2026-06-10T10:34:29.532Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"10dce8f4-b896-4c9e-9ea3-a5ddc2107633","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:29.532Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"10dce8f4-b896-4c9e-9ea3-a5ddc2107633","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:29.532Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"10dce8f4-b896-4c9e-9ea3-a5ddc2107633","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:29.532Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"10dce8f4-b896-4c9e-9ea3-a5ddc2107633"}
{"level":"info","ts":"2026-06-10T10:34:29.532Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"10dce8f4-b896-4c9e-9ea3-a5ddc2107633","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e"}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e"}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","podName":"counter-pc8bg-head-nwnwp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","podName":"counter-pc8bg-head-nwnwp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:29.943Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"8abe582b-8d87-48c4-ac97-4e121ad5696e","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:32.113Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51"}
{"level":"info","ts":"2026-06-10T10:34:32.113Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51"}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","podName":"counter-pc8bg-small-group-worker-w2rl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","podName":"counter-pc8bg-small-group-worker-w2rl6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:32.114Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"b104e58a-ef96-4714-a967-779d81018b51","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:32.533Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"690063ef-360d-4dda-b491-e6ccaa9a3c17","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:32.533Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"690063ef-360d-4dda-b491-e6ccaa9a3c17","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:32.533Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"690063ef-360d-4dda-b491-e6ccaa9a3c17","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:32.533Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"690063ef-360d-4dda-b491-e6ccaa9a3c17"}
{"level":"info","ts":"2026-06-10T10:34:32.533Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"690063ef-360d-4dda-b491-e6ccaa9a3c17","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0"}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0"}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","podName":"counter-pc8bg-head-nwnwp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","podName":"counter-pc8bg-head-nwnwp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:34:33.126Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5426b6e4-fe4a-4197-9545-8d505c75b7e0","name":"counter-pc8bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:33.140Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"4f894003-7492-4d8d-bf0a-07ed4f0a2db9","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:33.140Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"641b8d70-b2ed-41d1-a79f-80603b2432a6","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:33.140Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"4f894003-7492-4d8d-bf0a-07ed4f0a2db9","pod":"counter-pc8bg-head-nwnwp","ip":"10.134.0.22"}
{"level":"info","ts":"2026-06-10T10:34:33.140Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"4f894003-7492-4d8d-bf0a-07ed4f0a2db9","pod":"counter-pc8bg-small-group-worker-w2rl6","ip":"10.132.0.45"}
{"level":"info","ts":"2026-06-10T10:34:33.140Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f9f16b33-aa65-416e-a614-d39fa3c8c9b5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:33.140Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f9f16b33-aa65-416e-a614-d39fa3c8c9b5","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:33.140Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f9f16b33-aa65-416e-a614-d39fa3c8c9b5","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:33.140Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"4f894003-7492-4d8d-bf0a-07ed4f0a2db9","rayCluster":"counter-pc8bg","podIPs":["10.134.0.22","10.132.0.45"]}
{"level":"info","ts":"2026-06-10T10:34:33.141Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f9f16b33-aa65-416e-a614-d39fa3c8c9b5"}
{"level":"info","ts":"2026-06-10T10:34:33.141Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f9f16b33-aa65-416e-a614-d39fa3c8c9b5","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:33.157Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"641b8d70-b2ed-41d1-a79f-80603b2432a6","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:33.167Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"641b8d70-b2ed-41d1-a79f-80603b2432a6","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:33.168Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"641b8d70-b2ed-41d1-a79f-80603b2432a6","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b"}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b"}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","podName":"counter-pc8bg-head-nwnwp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:35.141Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","podName":"counter-pc8bg-head-nwnwp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:34:35.142Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:35.142Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"1dfb97d8-7275-4602-b0ac-e02546e18e4b","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:35.534Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ecc25000-f685-4bd4-a6a4-800f5042021e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:35.534Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ecc25000-f685-4bd4-a6a4-800f5042021e","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:35.534Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ecc25000-f685-4bd4-a6a4-800f5042021e","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:35.534Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ecc25000-f685-4bd4-a6a4-800f5042021e"}
{"level":"info","ts":"2026-06-10T10:34:35.534Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ecc25000-f685-4bd4-a6a4-800f5042021e","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:38.535Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"63a88751-742e-47e1-a0d1-d10b54a6c8dd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:38.535Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"63a88751-742e-47e1-a0d1-d10b54a6c8dd","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:38.535Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"63a88751-742e-47e1-a0d1-d10b54a6c8dd","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:38.535Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"63a88751-742e-47e1-a0d1-d10b54a6c8dd"}
{"level":"info","ts":"2026-06-10T10:34:38.535Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"63a88751-742e-47e1-a0d1-d10b54a6c8dd","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9"}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9"}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","podName":"counter-pc8bg-small-group-worker-w2rl6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","podName":"counter-pc8bg-small-group-worker-w2rl6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:34:38.963Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"672a0f36-bf11-4fcf-a610-6184f08778c9","name":"counter-pc8bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"5e010905-e59b-4fa7-abfd-9b9e558d8dce","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"f205f1f8-db17-4135-b94e-4e6a4e123c3d","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"5e010905-e59b-4fa7-abfd-9b9e558d8dce","pod":"counter-pc8bg-head-nwnwp","ip":"10.134.0.22"}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"5e010905-e59b-4fa7-abfd-9b9e558d8dce","pod":"counter-pc8bg-small-group-worker-w2rl6","ip":"10.132.0.45"}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"5e010905-e59b-4fa7-abfd-9b9e558d8dce","rayCluster":"counter-pc8bg","podIPs":["10.134.0.22","10.132.0.45"]}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"bd312bf4-e2cc-4124-ba3a-7167aac14c15","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"bd312bf4-e2cc-4124-ba3a-7167aac14c15","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"bd312bf4-e2cc-4124-ba3a-7167aac14c15","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"bd312bf4-e2cc-4124-ba3a-7167aac14c15"}
{"level":"info","ts":"2026-06-10T10:34:38.975Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"bd312bf4-e2cc-4124-ba3a-7167aac14c15","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:38.984Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"f205f1f8-db17-4135-b94e-4e6a4e123c3d","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:38.997Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"f205f1f8-db17-4135-b94e-4e6a4e123c3d","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:38.997Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"f205f1f8-db17-4135-b94e-4e6a4e123c3d","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:40.975Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0"}
{"level":"info","ts":"2026-06-10T10:34:40.975Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0"}
{"level":"info","ts":"2026-06-10T10:34:40.975Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","podName":"counter-pc8bg-small-group-worker-w2rl6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","podName":"counter-pc8bg-small-group-worker-w2rl6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:34:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:40.976Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"2eae6686-4c92-4d17-b09b-033fadd604e0","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:41.536Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"180eecff-98d8-4a69-9a74-c59acd993776","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:41.536Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"180eecff-98d8-4a69-9a74-c59acd993776","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:41.536Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"180eecff-98d8-4a69-9a74-c59acd993776","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:41.536Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"180eecff-98d8-4a69-9a74-c59acd993776"}
{"level":"info","ts":"2026-06-10T10:34:41.536Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"180eecff-98d8-4a69-9a74-c59acd993776","RayCluster":"counter-pc8bg","State":""}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41"}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41"}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:34:44.147Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"32aad0c8-d60c-4257-9550-b03352fffd41","name":"counter-pc8bg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:34:44.158Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e23c97af-8ee4-4b36-8070-08bdbad1367d","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"872adfc5-ed5a-4a21-a207-4d2e469e06f2","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e23c97af-8ee4-4b36-8070-08bdbad1367d","pod":"counter-pc8bg-head-nwnwp","ip":"10.134.0.22"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e23c97af-8ee4-4b36-8070-08bdbad1367d","pod":"counter-pc8bg-small-group-worker-w2rl6","ip":"10.132.0.45"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e23c97af-8ee4-4b36-8070-08bdbad1367d","rayCluster":"counter-pc8bg","podIPs":["10.134.0.22","10.132.0.45"]}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","head service name":"counter-pc8bg-head-svc","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","head service URL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9"}
{"level":"info","ts":"2026-06-10T10:34:44.159Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","command":["if","!","ray","job","status","--address","http://counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","counter-qdh2n",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-qdh2n","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","--follow","counter-qdh2n"]}
{"level":"info","ts":"2026-06-10T10:34:44.167Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-10T10:34:44.167Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","SubmissionMode":"K8sJobMode","RayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.167Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:34:44.167Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9ceacc2f-91b4-437c-ac38-1594aa1589f9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:34:44.170Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"872adfc5-ed5a-4a21-a207-4d2e469e06f2","name":"counter-pc8bg-head"}
{"level":"info","ts":"2026-06-10T10:34:44.177Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:44.177Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.177Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:44.178Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273"}
{"level":"info","ts":"2026-06-10T10:34:44.178Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","head service name":"counter-pc8bg-head-svc","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:34:44.178Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","head service URL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:34:44.178Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-10T10:34:44.178Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","SubmissionMode":"K8sJobMode","RayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.178Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:34:44.178Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:34:44.199Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"872adfc5-ed5a-4a21-a207-4d2e469e06f2","name":"counter-pc8bg-workers"}
{"level":"info","ts":"2026-06-10T10:34:44.199Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-10T10:34:44.199Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"872adfc5-ed5a-4a21-a207-4d2e469e06f2","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.199Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273"}
{"level":"error","ts":"2026-06-10T10:34:44.199Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"42574fcf-b449-42f0-b9cb-8ff97ebf8273","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.199Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8b76b48f-1a55-4eff-a298-13f020f3a753","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:44.199Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8b76b48f-1a55-4eff-a298-13f020f3a753","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.199Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8b76b48f-1a55-4eff-a298-13f020f3a753","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:44.199Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8b76b48f-1a55-4eff-a298-13f020f3a753"}
{"level":"error","ts":"2026-06-10T10:34:44.211Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8b76b48f-1a55-4eff-a298-13f020f3a753","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.211Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8b76b48f-1a55-4eff-a298-13f020f3a753"}
{"level":"error","ts":"2026-06-10T10:34:44.211Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8b76b48f-1a55-4eff-a298-13f020f3a753","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.211Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"0f8ca800-88e0-4a9b-adde-5d15682e369a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:44.211Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"0f8ca800-88e0-4a9b-adde-5d15682e369a","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.211Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"0f8ca800-88e0-4a9b-adde-5d15682e369a","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:44.211Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"0f8ca800-88e0-4a9b-adde-5d15682e369a"}
{"level":"error","ts":"2026-06-10T10:34:44.216Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"0f8ca800-88e0-4a9b-adde-5d15682e369a","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.216Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"0f8ca800-88e0-4a9b-adde-5d15682e369a"}
{"level":"error","ts":"2026-06-10T10:34:44.216Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"0f8ca800-88e0-4a9b-adde-5d15682e369a","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.222Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"006fbe30-337c-408a-b875-0616e93aac89","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:44.222Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"006fbe30-337c-408a-b875-0616e93aac89","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.222Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"006fbe30-337c-408a-b875-0616e93aac89","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:44.222Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"006fbe30-337c-408a-b875-0616e93aac89"}
{"level":"error","ts":"2026-06-10T10:34:44.226Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"006fbe30-337c-408a-b875-0616e93aac89","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.226Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"006fbe30-337c-408a-b875-0616e93aac89"}
{"level":"error","ts":"2026-06-10T10:34:44.226Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"006fbe30-337c-408a-b875-0616e93aac89","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.267Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6948cbf4-43be-4eea-9ba9-20cb404a2f0f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:44.267Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6948cbf4-43be-4eea-9ba9-20cb404a2f0f","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.267Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6948cbf4-43be-4eea-9ba9-20cb404a2f0f","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:44.267Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6948cbf4-43be-4eea-9ba9-20cb404a2f0f"}
{"level":"error","ts":"2026-06-10T10:34:44.272Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6948cbf4-43be-4eea-9ba9-20cb404a2f0f","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.272Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6948cbf4-43be-4eea-9ba9-20cb404a2f0f"}
{"level":"error","ts":"2026-06-10T10:34:44.272Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"6948cbf4-43be-4eea-9ba9-20cb404a2f0f","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.352Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"933d9f73-79b5-4779-b6c2-fc8e41fd03ab","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:44.352Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"933d9f73-79b5-4779-b6c2-fc8e41fd03ab","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.352Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"933d9f73-79b5-4779-b6c2-fc8e41fd03ab","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:44.352Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"933d9f73-79b5-4779-b6c2-fc8e41fd03ab"}
{"level":"error","ts":"2026-06-10T10:34:44.358Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"933d9f73-79b5-4779-b6c2-fc8e41fd03ab","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.358Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"933d9f73-79b5-4779-b6c2-fc8e41fd03ab"}
{"level":"error","ts":"2026-06-10T10:34:44.358Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"933d9f73-79b5-4779-b6c2-fc8e41fd03ab","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.518Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ef23c3fa-ca55-4ff8-a470-5b9eb22aeb0a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:44.518Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ef23c3fa-ca55-4ff8-a470-5b9eb22aeb0a","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.518Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ef23c3fa-ca55-4ff8-a470-5b9eb22aeb0a","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:44.518Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ef23c3fa-ca55-4ff8-a470-5b9eb22aeb0a"}
{"level":"error","ts":"2026-06-10T10:34:44.524Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ef23c3fa-ca55-4ff8-a470-5b9eb22aeb0a","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.524Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ef23c3fa-ca55-4ff8-a470-5b9eb22aeb0a"}
{"level":"error","ts":"2026-06-10T10:34:44.524Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ef23c3fa-ca55-4ff8-a470-5b9eb22aeb0a","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7d1801a5-83f1-4800-bcf3-12a0a0cc3d41","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:44.844Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7d1801a5-83f1-4800-bcf3-12a0a0cc3d41","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:44.844Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7d1801a5-83f1-4800-bcf3-12a0a0cc3d41","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:44.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7d1801a5-83f1-4800-bcf3-12a0a0cc3d41"}
{"level":"error","ts":"2026-06-10T10:34:44.850Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7d1801a5-83f1-4800-bcf3-12a0a0cc3d41","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:44.850Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7d1801a5-83f1-4800-bcf3-12a0a0cc3d41"}
{"level":"error","ts":"2026-06-10T10:34:44.850Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7d1801a5-83f1-4800-bcf3-12a0a0cc3d41","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:45.188Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4c06aacc-5d88-4088-b872-80fa484c176a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:45.188Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4c06aacc-5d88-4088-b872-80fa484c176a","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:45.188Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4c06aacc-5d88-4088-b872-80fa484c176a","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:45.188Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4c06aacc-5d88-4088-b872-80fa484c176a"}
{"level":"error","ts":"2026-06-10T10:34:45.194Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4c06aacc-5d88-4088-b872-80fa484c176a","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:45.194Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4c06aacc-5d88-4088-b872-80fa484c176a"}
{"level":"error","ts":"2026-06-10T10:34:45.194Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"4c06aacc-5d88-4088-b872-80fa484c176a","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:45.490Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f28632be-a267-4b09-823f-84d246962016","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:45.490Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f28632be-a267-4b09-823f-84d246962016","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:45.490Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f28632be-a267-4b09-823f-84d246962016","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:45.491Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f28632be-a267-4b09-823f-84d246962016"}
{"level":"error","ts":"2026-06-10T10:34:45.496Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f28632be-a267-4b09-823f-84d246962016","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:45.496Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f28632be-a267-4b09-823f-84d246962016"}
{"level":"error","ts":"2026-06-10T10:34:45.496Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"f28632be-a267-4b09-823f-84d246962016","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:46.158Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b"}
{"level":"info","ts":"2026-06-10T10:34:46.158Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b"}
{"level":"info","ts":"2026-06-10T10:34:46.158Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","1 head service found":"counter-pc8bg-head-svc"}
{"level":"info","ts":"2026-06-10T10:34:46.158Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","Found 1 head Pod":"counter-pc8bg-head-nwnwp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:34:46.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","head Pod":"counter-pc8bg-head-nwnwp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-pc8bg-head-nwnwp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:46.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:34:46.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","worker Pod":"counter-pc8bg-small-group-worker-w2rl6","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-pc8bg-small-group-worker-w2rl6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:34:46.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:46.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:34:46.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:34:46.159Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:34:46.159Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"50ca0177-0b87-4320-aca9-6a51e494082b","seconds":300}
{"level":"info","ts":"2026-06-10T10:34:46.911Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"981a9502-4374-4ceb-9f09-57fa8263cb75","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:34:46.911Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"981a9502-4374-4ceb-9f09-57fa8263cb75","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:34:46.911Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"981a9502-4374-4ceb-9f09-57fa8263cb75","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:34:46.912Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"981a9502-4374-4ceb-9f09-57fa8263cb75","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:34:48.057Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"a9f03c6c-82d0-448c-9ca7-2d784c7e6fb5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:48.057Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"a9f03c6c-82d0-448c-9ca7-2d784c7e6fb5","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:48.057Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"a9f03c6c-82d0-448c-9ca7-2d784c7e6fb5","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:48.057Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"a9f03c6c-82d0-448c-9ca7-2d784c7e6fb5"}
{"level":"error","ts":"2026-06-10T10:34:48.063Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"a9f03c6c-82d0-448c-9ca7-2d784c7e6fb5","JobId":"counter-qdh2n","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:48.063Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"a9f03c6c-82d0-448c-9ca7-2d784c7e6fb5"}
{"level":"error","ts":"2026-06-10T10:34:48.063Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"a9f03c6c-82d0-448c-9ca7-2d784c7e6fb5","error":"Job counter-qdh2n does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:34:48.716Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"005e9370-c4da-41f1-ac30-c153d1462311","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:48.717Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"005e9370-c4da-41f1-ac30-c153d1462311","pod":"counter-pc8bg-head-nwnwp","ip":"10.134.0.22"}
{"level":"info","ts":"2026-06-10T10:34:48.717Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"005e9370-c4da-41f1-ac30-c153d1462311","pod":"counter-pc8bg-small-group-worker-w2rl6","ip":"10.132.0.45"}
{"level":"info","ts":"2026-06-10T10:34:48.717Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"005e9370-c4da-41f1-ac30-c153d1462311","rayCluster":"counter-pc8bg","podIPs":["10.134.0.22","10.132.0.45"]}
{"level":"info","ts":"2026-06-10T10:34:53.184Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e06f4a20-ac4c-4c1f-b703-31acf208ecf9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:53.184Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e06f4a20-ac4c-4c1f-b703-31acf208ecf9","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:53.184Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e06f4a20-ac4c-4c1f-b703-31acf208ecf9","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:53.184Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e06f4a20-ac4c-4c1f-b703-31acf208ecf9"}
{"level":"info","ts":"2026-06-10T10:34:53.191Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e06f4a20-ac4c-4c1f-b703-31acf208ecf9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:34:53.191Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"e06f4a20-ac4c-4c1f-b703-31acf208ecf9","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:34:53.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"79c91297-6bb5-4797-9301-27e380cdfae6","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:53.205Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"79c91297-6bb5-4797-9301-27e380cdfae6","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:53.205Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"79c91297-6bb5-4797-9301-27e380cdfae6","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:53.205Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"79c91297-6bb5-4797-9301-27e380cdfae6"}
{"level":"info","ts":"2026-06-10T10:34:53.208Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"79c91297-6bb5-4797-9301-27e380cdfae6","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:34:56.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"864e6386-80e6-4652-b212-e058f62cddf4","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:56.205Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"864e6386-80e6-4652-b212-e058f62cddf4","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:56.205Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"864e6386-80e6-4652-b212-e058f62cddf4","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:56.205Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"864e6386-80e6-4652-b212-e058f62cddf4"}
{"level":"info","ts":"2026-06-10T10:34:56.209Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"864e6386-80e6-4652-b212-e058f62cddf4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:34:56.209Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"864e6386-80e6-4652-b212-e058f62cddf4","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:34:56.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9098d49a-4c24-4667-ae8e-e448b6c002c0","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:56.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9098d49a-4c24-4667-ae8e-e448b6c002c0","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:56.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9098d49a-4c24-4667-ae8e-e448b6c002c0","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:56.223Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9098d49a-4c24-4667-ae8e-e448b6c002c0"}
{"level":"info","ts":"2026-06-10T10:34:56.225Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9098d49a-4c24-4667-ae8e-e448b6c002c0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:34:59.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8754fca3-0727-4af7-b222-99ad973f0770","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:34:59.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8754fca3-0727-4af7-b222-99ad973f0770","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:34:59.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8754fca3-0727-4af7-b222-99ad973f0770","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:34:59.223Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8754fca3-0727-4af7-b222-99ad973f0770"}
{"level":"info","ts":"2026-06-10T10:34:59.227Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"8754fca3-0727-4af7-b222-99ad973f0770","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:02.066Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"83aae039-7114-43df-84cf-4207cf89e033","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:02.066Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"83aae039-7114-43df-84cf-4207cf89e033","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:02.066Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"83aae039-7114-43df-84cf-4207cf89e033","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:02.066Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"83aae039-7114-43df-84cf-4207cf89e033"}
{"level":"info","ts":"2026-06-10T10:35:02.070Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"83aae039-7114-43df-84cf-4207cf89e033","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:02.227Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"627d0e24-09d0-4500-ad26-97f5828b6521","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:02.227Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"627d0e24-09d0-4500-ad26-97f5828b6521","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:02.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"627d0e24-09d0-4500-ad26-97f5828b6521","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:02.227Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"627d0e24-09d0-4500-ad26-97f5828b6521"}
{"level":"info","ts":"2026-06-10T10:35:02.231Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"627d0e24-09d0-4500-ad26-97f5828b6521","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:03.074Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ad51c371-a58a-491c-8215-b030fdf886a1","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:03.074Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ad51c371-a58a-491c-8215-b030fdf886a1","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.074Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ad51c371-a58a-491c-8215-b030fdf886a1","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:03.074Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ad51c371-a58a-491c-8215-b030fdf886a1"}
{"level":"info","ts":"2026-06-10T10:35:03.078Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"ad51c371-a58a-491c-8215-b030fdf886a1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:03.091Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7fd2a633-f24e-45ec-844c-8784f9b649c3","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:03.091Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7fd2a633-f24e-45ec-844c-8784f9b649c3","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.091Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7fd2a633-f24e-45ec-844c-8784f9b649c3","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:03.091Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7fd2a633-f24e-45ec-844c-8784f9b649c3"}
{"level":"info","ts":"2026-06-10T10:35:03.094Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7fd2a633-f24e-45ec-844c-8784f9b649c3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:34:51Z","endTime":"2026-06-10T10:34:55Z"},"jobId":"counter-qdh2n","rayClusterName":"counter-pc8bg","dashboardURL":"counter-pc8bg-head-svc.test-ns-nrv9s.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-10T10:34:18Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:34:44Z","stateTransitionTimes":{"ready":"2026-06-10T10:34:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.22","serviceIP":"10.134.0.22","podName":"counter-pc8bg-head-nwnwp","serviceName":"counter-pc8bg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:34:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:34:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:34:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:34:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:03.094Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"7fd2a633-f24e-45ec-844c-8784f9b649c3","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-10T10:35:03.107Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9dc9e753-704c-4527-80cf-9fc6b1c7b8cd","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:03.107Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9dc9e753-704c-4527-80cf-9fc6b1c7b8cd","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:35:03 +0000 UTC","Now":"2026-06-10T10:35:03.107Z","ShutdownTime":"2026-06-10T10:35:03.000Z"}
{"level":"info","ts":"2026-06-10T10:35:03.107Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9dc9e753-704c-4527-80cf-9fc6b1c7b8cd","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:35:03.116Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9dc9e753-704c-4527-80cf-9fc6b1c7b8cd","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:03.116Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9dc9e753-704c-4527-80cf-9fc6b1c7b8cd","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:35:03.116Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"9dc9e753-704c-4527-80cf-9fc6b1c7b8cd","RayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.116Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6991d03c-02fb-4801-ac52-86850f887caf","namespacedName":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:03.116Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"reconcileID":"5ebdb85a-8e7d-4b7a-bf3c-026247bf8443"}
{"level":"info","ts":"2026-06-10T10:35:03.116Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6991d03c-02fb-4801-ac52-86850f887caf","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.116Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6991d03c-02fb-4801-ac52-86850f887caf","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.116Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"4ffd47de-d3c6-4b8d-95be-784c909df3e4"}
{"level":"info","ts":"2026-06-10T10:35:03.116Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"bfb768ca-07a4-4cf9-9cfd-64bfea802820","rayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6991d03c-02fb-4801-ac52-86850f887caf","namespace":"test-ns-nrv9s","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"bfb768ca-07a4-4cf9-9cfd-64bfea802820","pod":"counter-pc8bg-head-nwnwp","ip":"10.134.0.22"}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"bfb768ca-07a4-4cf9-9cfd-64bfea802820","pod":"counter-pc8bg-small-group-worker-w2rl6","ip":"10.132.0.45"}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"bfb768ca-07a4-4cf9-9cfd-64bfea802820","rayCluster":"counter-pc8bg","podIPs":["10.134.0.22","10.132.0.45"]}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"349e1057-71e6-4586-b6cd-b821fffd8162","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"349e1057-71e6-4586-b6cd-b821fffd8162","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:35:03 +0000 UTC","Now":"2026-06-10T10:35:03.117Z","ShutdownTime":"2026-06-10T10:35:03.000Z"}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"349e1057-71e6-4586-b6cd-b821fffd8162","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"349e1057-71e6-4586-b6cd-b821fffd8162","RayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"349e1057-71e6-4586-b6cd-b821fffd8162","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:35:03.117Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"349e1057-71e6-4586-b6cd-b821fffd8162","RayCluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.122Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6991d03c-02fb-4801-ac52-86850f887caf","name":"kuberay-gateway-access","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:35:03.126Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6991d03c-02fb-4801-ac52-86850f887caf","name":"test-ns-nrv9s-counter-pc8bg","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:35:03.130Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6991d03c-02fb-4801-ac52-86850f887caf","configMap":"kube-rbac-proxy-config-counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.134Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6991d03c-02fb-4801-ac52-86850f887caf","serviceAccount":"counter-pc8bg-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:35:03.141Z","logger":"raycluster-resource","msg":"default","name":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.141Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-pc8bg","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"6991d03c-02fb-4801-ac52-86850f887caf","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"8e804587-ff7f-45b6-8c8c-1c7d785246c0","namespacedName":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"8e804587-ff7f-45b6-8c8c-1c7d785246c0","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"8e804587-ff7f-45b6-8c8c-1c7d785246c0","cluster":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"8e804587-ff7f-45b6-8c8c-1c7d785246c0","namespace":"test-ns-nrv9s","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"8e804587-ff7f-45b6-8c8c-1c7d785246c0","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"d5746e8a-e533-471d-9843-be45d986977f","namespace":"test-ns-nrv9s","clusterName":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"be23a8dd-e388-49b1-b80d-528af760c760"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"db140445-4e45-4052-8cd6-1119ffe4f058","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"db140445-4e45-4052-8cd6-1119ffe4f058","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:35:03 +0000 UTC","Now":"2026-06-10T10:35:03.150Z","ShutdownTime":"2026-06-10T10:35:03.000Z"}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"db140445-4e45-4052-8cd6-1119ffe4f058","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"db140445-4e45-4052-8cd6-1119ffe4f058","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"db140445-4e45-4052-8cd6-1119ffe4f058","isClusterDeleted":true}
{"level":"info","ts":"2026-06-10T10:35:03.150Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"db140445-4e45-4052-8cd6-1119ffe4f058","RayCluster":"counter-pc8bg"}
{"level":"error","ts":"2026-06-10T10:35:03.153Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"8e804587-ff7f-45b6-8c8c-1c7d785246c0","error":"rayclusters.ray.io \"counter-pc8bg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:03.153Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"8e804587-ff7f-45b6-8c8c-1c7d785246c0"}
{"level":"error","ts":"2026-06-10T10:35:03.153Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"8e804587-ff7f-45b6-8c8c-1c7d785246c0","error":"rayclusters.ray.io \"counter-pc8bg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:03.154Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e7693701-fc12-419e-88c6-d74c6604388f","namespacedName":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:03.154Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e7693701-fc12-419e-88c6-d74c6604388f"}
{"level":"info","ts":"2026-06-10T10:35:03.154Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e7693701-fc12-419e-88c6-d74c6604388f","namespace":"test-ns-nrv9s","authClusters":0}
{"level":"info","ts":"2026-06-10T10:35:03.154Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e7693701-fc12-419e-88c6-d74c6604388f","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:35:03.158Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"2583da66-ce28-4005-b521-7928d0e24241"}
{"level":"error","ts":"2026-06-10T10:35:03.170Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"d5746e8a-e533-471d-9843-be45d986977f","issuer":"ray-selfsigned-issuer-counter-pc8bg","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-pc8bg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:03.170Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e81c981a-e771-4ae2-ae5d-4ea236846625","namespacedName":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:03.199Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"59190af9-fdd2-4b53-93b8-d53b4c06b35b"}
{"level":"info","ts":"2026-06-10T10:35:03.199Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e81c981a-e771-4ae2-ae5d-4ea236846625"}
{"level":"info","ts":"2026-06-10T10:35:03.199Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e81c981a-e771-4ae2-ae5d-4ea236846625","namespace":"test-ns-nrv9s","authClusters":0}
{"level":"info","ts":"2026-06-10T10:35:03.199Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"e81c981a-e771-4ae2-ae5d-4ea236846625","namespace":"test-ns-nrv9s"}
{"level":"info","ts":"2026-06-10T10:35:03.199Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"59e7d0fc-8a93-4052-853a-f40d7277dfb5","namespacedName":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"}}
{"level":"info","ts":"2026-06-10T10:35:03.199Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"59e7d0fc-8a93-4052-853a-f40d7277dfb5"}
{"level":"info","ts":"2026-06-10T10:35:03.199Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"59e7d0fc-8a93-4052-853a-f40d7277dfb5","namespace":"test-ns-nrv9s","authClusters":0}
{"level":"info","ts":"2026-06-10T10:35:03.199Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"59e7d0fc-8a93-4052-853a-f40d7277dfb5","namespace":"test-ns-nrv9s"}
{"level":"error","ts":"2026-06-10T10:35:03.202Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"d5746e8a-e533-471d-9843-be45d986977f","certificate":"ray-head-cert-counter-pc8bg","error":"certificates.cert-manager.io \"ray-head-cert-counter-pc8bg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:03.217Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"d5746e8a-e533-471d-9843-be45d986977f","errorCount":2}
{"level":"info","ts":"2026-06-10T10:35:03.217Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"d5746e8a-e533-471d-9843-be45d986977f"}
{"level":"error","ts":"2026-06-10T10:35:03.217Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"d5746e8a-e533-471d-9843-be45d986977f","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-counter-pc8bg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:03.222Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"41a69161-5c96-475a-86d8-1ef04a67f17e","namespace":"test-ns-nrv9s","clusterName":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.222Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-pc8bg","namespace":"test-ns-nrv9s"},"namespace":"test-ns-nrv9s","name":"counter-pc8bg","reconcileID":"41a69161-5c96-475a-86d8-1ef04a67f17e","namespace":"test-ns-nrv9s","clusterName":"counter-pc8bg"}
{"level":"info","ts":"2026-06-10T10:35:03.806Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"215da134-1797-4c81-b40e-86bd91012a1d","DeletionTimestamp":"2026-06-10 10:35:03 +0000 UTC"}
{"level":"info","ts":"2026-06-10T10:35:03.806Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"215da134-1797-4c81-b40e-86bd91012a1d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:35:03.818Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"03783528-951e-4e16-8bc8-b949be52625d"}
{"level":"info","ts":"2026-06-10T10:35:03.823Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"fca06207-ff33-479e-9c40-e4a749bf7a79"}
{"level":"info","ts":"2026-06-10T10:35:05.232Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-nrv9s"},"reconcileID":"1da6888c-bdb9-4808-892c-d68107d62cc0"}
{"level":"info","ts":"2026-06-10T10:35:05.884Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3f61cfa1-7fa0-4fd5-9ae4-30417a6b59b6","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:05.884Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3f61cfa1-7fa0-4fd5-9ae4-30417a6b59b6","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:35:05.894Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3f61cfa1-7fa0-4fd5-9ae4-30417a6b59b6"}
{"level":"info","ts":"2026-06-10T10:35:05.894Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3f61cfa1-7fa0-4fd5-9ae4-30417a6b59b6","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:35:05.895Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3f61cfa1-7fa0-4fd5-9ae4-30417a6b59b6","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:35:05.895Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3f61cfa1-7fa0-4fd5-9ae4-30417a6b59b6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:35:05.905Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2a55f6a3-8ecc-4b1e-b716-eef8ace44a55","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:05.905Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2a55f6a3-8ecc-4b1e-b716-eef8ace44a55"}
{"level":"info","ts":"2026-06-10T10:35:05.905Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2a55f6a3-8ecc-4b1e-b716-eef8ace44a55","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:35:05.905Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2a55f6a3-8ecc-4b1e-b716-eef8ace44a55","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-695z2","rayClusterName":"counter-f5n48","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:35:05.905Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2a55f6a3-8ecc-4b1e-b716-eef8ace44a55","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:35:05.910Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2a55f6a3-8ecc-4b1e-b716-eef8ace44a55","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-10T10:35:05.910Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2a55f6a3-8ecc-4b1e-b716-eef8ace44a55"}
{"level":"error","ts":"2026-06-10T10:35:05.910Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2a55f6a3-8ecc-4b1e-b716-eef8ace44a55","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:05.910Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"a96562c0-f0dd-4e9a-89eb-44aa403c1f0f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:05.910Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"a96562c0-f0dd-4e9a-89eb-44aa403c1f0f","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.910Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"a96562c0-f0dd-4e9a-89eb-44aa403c1f0f","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:05.910Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"a96562c0-f0dd-4e9a-89eb-44aa403c1f0f","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:05.914Z","logger":"raycluster-resource","msg":"default","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.914Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-ksbpl","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"a96562c0-f0dd-4e9a-89eb-44aa403c1f0f","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"a96562c0-f0dd-4e9a-89eb-44aa403c1f0f"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f2692c82-6a34-4da2-b812-9176621263ab","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"a96562c0-f0dd-4e9a-89eb-44aa403c1f0f","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","namespacedName":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","mode":"IntegratedOAuth","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"369d7562-6c44-40a4-9aa6-8fe10a7fd713","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"369d7562-6c44-40a4-9aa6-8fe10a7fd713","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"369d7562-6c44-40a4-9aa6-8fe10a7fd713","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"369d7562-6c44-40a4-9aa6-8fe10a7fd713"}
{"level":"info","ts":"2026-06-10T10:35:05.928Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"369d7562-6c44-40a4-9aa6-8fe10a7fd713","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:05.933Z","logger":"raycluster-resource","msg":"default","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.933Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-ksbpl","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:35:05.936Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","name":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:05.936Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","error":"head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"error","ts":"2026-06-10T10:35:05.936Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:05.936Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"info","ts":"2026-06-10T10:35:05.936Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:05.936Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:05.936Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:05.936Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:35:05.936Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","name":"counter-ksbpl","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:05.937Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.937Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.942Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b7e50232-ba18-4a02-beea-f6f272b6a79b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:05.942Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b7e50232-ba18-4a02-beea-f6f272b6a79b","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.942Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b7e50232-ba18-4a02-beea-f6f272b6a79b","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:05.942Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b7e50232-ba18-4a02-beea-f6f272b6a79b"}
{"level":"info","ts":"2026-06-10T10:35:05.942Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b7e50232-ba18-4a02-beea-f6f272b6a79b","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:05.944Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f2692c82-6a34-4da2-b812-9176621263ab","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.944Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"785823d0-d6d0-413f-9418-5f693d656401","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","rayCluster":"counter-ksbpl","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","name":"counter-ksbpl-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","name":"counter-ksbpl","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-ksbpl\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"counter-ksbpl"}}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937"}
{"level":"error","ts":"2026-06-10T10:35:05.999Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"c92dd0f3-d119-4cf9-b92c-6a863c748937","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","error":"head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"error","ts":"2026-06-10T10:35:05.999Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:05.999Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"info","ts":"2026-06-10T10:35:06.000Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.000Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.000Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.000Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:35:06.000Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","name":"counter-ksbpl","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:06.003Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"785823d0-d6d0-413f-9418-5f693d656401","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:06.004Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","name":"kuberay-gateway-access","namespace":"test-ns-k92wt","operation":"created"}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb"}
{"level":"error","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"ce730ff4-91ea-42f4-8698-71f39c09bbbb","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b"}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b"}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","error":"head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"error","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:35:06.010Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","name":"counter-ksbpl","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:06.011Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"7e58203d-0b54-4b94-a535-ea0a8df930d4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:06.011Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"7e58203d-0b54-4b94-a535-ea0a8df930d4","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.011Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"7e58203d-0b54-4b94-a535-ea0a8df930d4","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:06.011Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"7e58203d-0b54-4b94-a535-ea0a8df930d4"}
{"level":"info","ts":"2026-06-10T10:35:06.011Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"7e58203d-0b54-4b94-a535-ea0a8df930d4","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:06.012Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","name":"counter-ksbpl","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-ksbpl\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"counter-ksbpl"}}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b"}
{"level":"error","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"834381bb-5219-499c-9e5e-59580a09244b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","name":"test-ns-k92wt-counter-ksbpl","operation":"created"}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25"}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25"}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25","error":"head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"error","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25"}
{"level":"error","ts":"2026-06-10T10:35:06.099Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"1b4b0358-286e-46c8-95c4-cd796b488f25","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.103Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"785823d0-d6d0-413f-9418-5f693d656401","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:06.103Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"785823d0-d6d0-413f-9418-5f693d656401","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.103Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"3664a2e5-f9c0-43ee-ab7b-b181f8ded481","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.106Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","name":"kube-rbac-proxy-config-counter-ksbpl","operation":"created"}
{"level":"info","ts":"2026-06-10T10:35:06.109Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f"}
{"level":"info","ts":"2026-06-10T10:35:06.110Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","rayCluster":"counter-ksbpl","podIPs":[]}
{"level":"error","ts":"2026-06-10T10:35:06.113Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-ksbpl\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.113Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d"}
{"level":"error","ts":"2026-06-10T10:35:06.113Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0a5a2206-59ef-40bf-8bdb-f58196c2df8d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-ksbpl\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.113Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f87c1db7-05df-45f9-a4cb-5aa9e5bfab92","namespacedName":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:06.113Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f87c1db7-05df-45f9-a4cb-5aa9e5bfab92","mode":"IntegratedOAuth","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.113Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f87c1db7-05df-45f9-a4cb-5aa9e5bfab92","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.113Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"3664a2e5-f9c0-43ee-ab7b-b181f8ded481","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210"}
{"level":"info","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210"}
{"level":"info","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210","error":"head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"error","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"info","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210"}
{"level":"error","ts":"2026-06-10T10:35:06.120Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"f150d25a-e545-49f2-ae38-1e68f1cdd210","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.121Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f"}
{"level":"info","ts":"2026-06-10T10:35:06.121Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","rayCluster":"counter-ksbpl","podIPs":[]}
{"level":"error","ts":"2026-06-10T10:35:06.121Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-ksbpl\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.121Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f"}
{"level":"error","ts":"2026-06-10T10:35:06.121Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"99011f1e-177a-47f9-a50f-f634c08ede8f","error":"Certificate.cert-manager.io \"ray-worker-cert-counter-ksbpl\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.121Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"953981b8-b2df-4645-9158-8621f63c571d","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.124Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"3664a2e5-f9c0-43ee-ab7b-b181f8ded481","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:06.124Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"3664a2e5-f9c0-43ee-ab7b-b181f8ded481","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.125Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f87c1db7-05df-45f9-a4cb-5aa9e5bfab92","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.126Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"54f4f63b-5d77-482f-a538-802220e96059","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"294744a7-2cda-495c-b5bc-1ccb2e159d2c","namespacedName":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:06.126Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e09ef912-a89d-4fd0-93b6-6ed0eeefe053","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e09ef912-a89d-4fd0-93b6-6ed0eeefe053","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"294744a7-2cda-495c-b5bc-1ccb2e159d2c","mode":"IntegratedOAuth","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"294744a7-2cda-495c-b5bc-1ccb2e159d2c","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e09ef912-a89d-4fd0-93b6-6ed0eeefe053","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e09ef912-a89d-4fd0-93b6-6ed0eeefe053"}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e09ef912-a89d-4fd0-93b6-6ed0eeefe053","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"953981b8-b2df-4645-9158-8621f63c571d"}
{"level":"error","ts":"2026-06-10T10:35:06.199Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"953981b8-b2df-4645-9158-8621f63c571d","error":"certificates.cert-manager.io \"ray-worker-cert-counter-ksbpl\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createRayWorkerCertificate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:732\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:206\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:35:06.199Z","msg":"Failed to create Ray worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"953981b8-b2df-4645-9158-8621f63c571d","error":"certificates.cert-manager.io \"ray-worker-cert-counter-ksbpl\" already exists","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:207\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"953981b8-b2df-4645-9158-8621f63c571d"}
{"level":"error","ts":"2026-06-10T10:35:06.199Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"953981b8-b2df-4645-9158-8621f63c571d","error":"certificates.cert-manager.io \"ray-worker-cert-counter-ksbpl\" already exists","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.199Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"dd0ec03f-92f7-4e8b-8780-7eb2d6fa9c10","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.200Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"dd0ec03f-92f7-4e8b-8780-7eb2d6fa9c10"}
{"level":"info","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137"}
{"level":"info","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137"}
{"level":"info","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137","error":"head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"error","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"info","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137"}
{"level":"error","ts":"2026-06-10T10:35:06.201Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"b51804f0-6c63-4973-96de-ccedbbd9a137","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.207Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"294744a7-2cda-495c-b5bc-1ccb2e159d2c","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.208Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"54f4f63b-5d77-482f-a538-802220e96059","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:06.210Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"ff620f1e-9106-4a0e-a1a4-a46a5b79ae05","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.211Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"ff620f1e-9106-4a0e-a1a4-a46a5b79ae05"}
{"level":"info","ts":"2026-06-10T10:35:06.218Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"54f4f63b-5d77-482f-a538-802220e96059","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:06.218Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"54f4f63b-5d77-482f-a538-802220e96059","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a"}
{"level":"info","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a"}
{"level":"info","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a","error":"head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"error","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"info","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a"}
{"level":"error","ts":"2026-06-10T10:35:06.361Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8cda4ecb-b878-46a2-bec5-a8100879603a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf"}
{"level":"info","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf"}
{"level":"info","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf","error":"head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"error","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"info","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf"}
{"level":"error","ts":"2026-06-10T10:35:06.682Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"81d911b2-6a88-4941-bf88-938481915daf","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f"}
{"level":"info","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f"}
{"level":"info","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f","error":"head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"error","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found"}
{"level":"info","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f","filter labels":["test-ns-k92wt",{"ray.io/cluster":"counter-ksbpl","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f"}
{"level":"error","ts":"2026-06-10T10:35:07.323Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6e6ed97c-15c9-44b1-b785-b95a262b035f","error":"mTLS secrets not ready: head certificate secret ray-head-secret-counter-ksbpl not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289"}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289"}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289"}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289"}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","mode":"IntegratedOAuth","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","cluster":"counter-ksbpl","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","cluster":"counter-ksbpl","authType":"IntegratedOAuth","serviceAccount":"counter-ksbpl-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-ksbpl","ray.io/group":"headgroup","ray.io/identifier":"counter-ksbpl-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:35:08.604Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","enabled":true}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","name":"counter-ksbpl-head-827hc"}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289"}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","nodeType":"worker","rayStartParams":{"address":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","rayStartCmd":"ray start  --address=counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:35:08.621Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","enabled":true}
{"level":"info","ts":"2026-06-10T10:35:08.633Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","name":"counter-ksbpl-small-group-worker-lh7wz"}
{"level":"info","ts":"2026-06-10T10:35:08.633Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.633Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.633Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-ksbpl-head-svc"},"newHead":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"}}
{"level":"info","ts":"2026-06-10T10:35:08.633Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"d2be766c-1568-401e-afc0-e6e73b19b289","name":"counter-ksbpl","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43"}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43"}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"counter-ksbpl-head-svc"},"newHead":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"}}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","name":"counter-ksbpl","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f4be65c2-c854-4cb0-b11e-5332154a314b","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.646Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"6b76fe90-996f-42f3-97e7-a4a6d7bfdb6a","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.647Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cf76b344-9c36-4b45-aa16-5b143efdde8d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:08.647Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cf76b344-9c36-4b45-aa16-5b143efdde8d","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.647Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cf76b344-9c36-4b45-aa16-5b143efdde8d","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:08.647Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cf76b344-9c36-4b45-aa16-5b143efdde8d"}
{"level":"info","ts":"2026-06-10T10:35:08.647Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cf76b344-9c36-4b45-aa16-5b143efdde8d","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:08.658Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f4be65c2-c854-4cb0-b11e-5332154a314b","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:08.669Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f4be65c2-c854-4cb0-b11e-5332154a314b","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:08.669Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f4be65c2-c854-4cb0-b11e-5332154a314b","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"6b76fe90-996f-42f3-97e7-a4a6d7bfdb6a","rayCluster":"counter-ksbpl","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","name":"counter-ksbpl","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-ksbpl\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"counter-ksbpl"}}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43"}
{"level":"error","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"8aeb9e61-7a3a-4e3d-90a2-362b0cdf0d43","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-ksbpl\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:35:08.699Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"cd308d57-7867-4a6f-b092-53269f02cb70","name":"counter-ksbpl","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:08.710Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","podName":"counter-ksbpl-head-827hc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","podName":"counter-ksbpl-head-827hc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d3f56596-5a68-468f-a1c5-ac52c1a7c5ff","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fabc37b8-0c30-442c-b8a6-d7e591a3603f","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"fe34137b-ac1e-41e1-b640-016dec4958e5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"fe34137b-ac1e-41e1-b640-016dec4958e5","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"fe34137b-ac1e-41e1-b640-016dec4958e5","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","name":"counter-ksbpl","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fabc37b8-0c30-442c-b8a6-d7e591a3603f","rayCluster":"counter-ksbpl","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"fe34137b-ac1e-41e1-b640-016dec4958e5"}
{"level":"info","ts":"2026-06-10T10:35:08.711Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"fe34137b-ac1e-41e1-b640-016dec4958e5","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:08.717Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","name":"counter-ksbpl","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-ksbpl\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"counter-ksbpl"}}
{"level":"info","ts":"2026-06-10T10:35:08.717Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08"}
{"level":"error","ts":"2026-06-10T10:35:08.717Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"38e4361a-d0ea-479c-9060-bde357cd3f08","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-ksbpl\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:08.720Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d3f56596-5a68-468f-a1c5-ac52c1a7c5ff","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17"}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17"}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:08.723Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"4c3a965d-da67-4252-94d4-a2c5cd0a9e17","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:08.730Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d3f56596-5a68-468f-a1c5-ac52c1a7c5ff","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:08.730Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d3f56596-5a68-468f-a1c5-ac52c1a7c5ff","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.929Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5e7e8366-902f-4f38-88e4-431b1f264cdc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:08.929Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5e7e8366-902f-4f38-88e4-431b1f264cdc","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:08.929Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5e7e8366-902f-4f38-88e4-431b1f264cdc","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:08.929Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5e7e8366-902f-4f38-88e4-431b1f264cdc"}
{"level":"info","ts":"2026-06-10T10:35:08.929Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5e7e8366-902f-4f38-88e4-431b1f264cdc","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d"}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d"}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:09.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:09.073Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","podName":"counter-ksbpl-head-827hc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:09.073Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","podName":"counter-ksbpl-head-827hc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:09.073Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:09.073Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"69b657be-4c5e-4883-a213-9d75b61a893d","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:09.101Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805"}
{"level":"info","ts":"2026-06-10T10:35:09.101Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805"}
{"level":"info","ts":"2026-06-10T10:35:09.101Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:09.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:09.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:09.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:09.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:09.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:09.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:09.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:09.102Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:09.102Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:09.103Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:09.103Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e64b791b-2bc6-4bb3-a501-794f11ff7805","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6"}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6"}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:09.328Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"25fb85d4-b9fa-460f-8f89-b5f047b39cd6","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f"}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f"}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","podName":"counter-ksbpl-head-827hc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","podName":"counter-ksbpl-head-827hc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"newHead":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"}}
{"level":"info","ts":"2026-06-10T10:35:10.096Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"dcceb3c8-0ca6-4f2c-b94d-fe8f5d58957f","name":"counter-ksbpl","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:10.114Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"a0526a6e-38e7-47e0-9868-cb7c39f38b48","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:10.114Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8139e887-0c67-4ea6-8897-a572049b6e85","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:10.114Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8139e887-0c67-4ea6-8897-a572049b6e85","pod":"counter-ksbpl-head-827hc","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-10T10:35:10.114Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8139e887-0c67-4ea6-8897-a572049b6e85","pod":"counter-ksbpl-small-group-worker-lh7wz","ip":"10.132.0.46"}
{"level":"info","ts":"2026-06-10T10:35:10.114Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c52c8d0-96a9-49e1-b940-b55ecb3c49ed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:10.114Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c52c8d0-96a9-49e1-b940-b55ecb3c49ed","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:10.114Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c52c8d0-96a9-49e1-b940-b55ecb3c49ed","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:10.114Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c52c8d0-96a9-49e1-b940-b55ecb3c49ed"}
{"level":"info","ts":"2026-06-10T10:35:10.114Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c52c8d0-96a9-49e1-b940-b55ecb3c49ed","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:10.128Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"a0526a6e-38e7-47e0-9868-cb7c39f38b48","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:10.129Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8139e887-0c67-4ea6-8897-a572049b6e85","certificate":"ray-head-cert-counter-ksbpl","dnsNames":["counter-ksbpl-head-svc","localhost","counter-ksbpl-head-svc.test-ns-k92wt.svc","counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.24","10.132.0.46","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:35:10.145Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8139e887-0c67-4ea6-8897-a572049b6e85","certificate":"ray-worker-cert-counter-ksbpl","dnsNames":["counter-ksbpl-worker-svc","localhost","counter-ksbpl-small-group","counter-ksbpl-worker-svc.test-ns-k92wt.svc","counter-ksbpl-worker-svc.test-ns-k92wt.svc.cluster.local","*.counter-ksbpl-worker-svc.test-ns-k92wt.svc","*.counter-ksbpl-worker-svc.test-ns-k92wt.svc.cluster.local","*-worker-*.test-ns-k92wt.svc","*-worker-*.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.24","10.132.0.46","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:35:10.145Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8139e887-0c67-4ea6-8897-a572049b6e85","rayCluster":"counter-ksbpl","podIPs":["10.134.0.24","10.132.0.46"]}
{"level":"info","ts":"2026-06-10T10:35:10.150Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"a0526a6e-38e7-47e0-9868-cb7c39f38b48","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:10.151Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"a0526a6e-38e7-47e0-9868-cb7c39f38b48","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:11.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c3ee764-aef7-4c78-8a06-a7efdf45e33a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:11.930Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c3ee764-aef7-4c78-8a06-a7efdf45e33a","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:11.930Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c3ee764-aef7-4c78-8a06-a7efdf45e33a","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:11.930Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c3ee764-aef7-4c78-8a06-a7efdf45e33a"}
{"level":"info","ts":"2026-06-10T10:35:11.930Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"2c3ee764-aef7-4c78-8a06-a7efdf45e33a","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732"}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732"}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:12.114Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:12.115Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:12.115Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"9b7815c3-cc86-4ded-865b-e796d63e3732","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:14.931Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"29f874c1-675e-4c98-b6e9-a39479ae6738","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:14.931Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"29f874c1-675e-4c98-b6e9-a39479ae6738","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:14.931Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"29f874c1-675e-4c98-b6e9-a39479ae6738","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:14.931Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"29f874c1-675e-4c98-b6e9-a39479ae6738"}
{"level":"info","ts":"2026-06-10T10:35:14.931Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"29f874c1-675e-4c98-b6e9-a39479ae6738","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29"}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29"}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","podName":"counter-ksbpl-head-827hc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:15.121Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","podName":"counter-ksbpl-head-827hc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:35:15.122Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:15.122Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"959928c1-7d5d-4d0d-8370-b6dc0a833b29","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9"}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9"}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:35:16.122Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"223757d7-8b01-49ed-8224-aee8cf4fddb9","name":"counter-ksbpl","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"4b894bc7-028a-42bd-8802-7568717790a3","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"6165f04f-7110-447c-a6d6-9d556aaba35d","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"4b894bc7-028a-42bd-8802-7568717790a3","pod":"counter-ksbpl-head-827hc","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"4b894bc7-028a-42bd-8802-7568717790a3","pod":"counter-ksbpl-small-group-worker-lh7wz","ip":"10.132.0.46"}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"06dde019-294b-41fd-ad28-fef22b4f21e5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"06dde019-294b-41fd-ad28-fef22b4f21e5","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"06dde019-294b-41fd-ad28-fef22b4f21e5","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"4b894bc7-028a-42bd-8802-7568717790a3","rayCluster":"counter-ksbpl","podIPs":["10.134.0.24","10.132.0.46"]}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"06dde019-294b-41fd-ad28-fef22b4f21e5"}
{"level":"info","ts":"2026-06-10T10:35:16.134Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"06dde019-294b-41fd-ad28-fef22b4f21e5","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:16.143Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"6165f04f-7110-447c-a6d6-9d556aaba35d","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:16.151Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"6165f04f-7110-447c-a6d6-9d556aaba35d","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:16.151Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"6165f04f-7110-447c-a6d6-9d556aaba35d","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:17.932Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1647bbb6-0076-4c69-9b17-dc39f795db13","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:17.932Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1647bbb6-0076-4c69-9b17-dc39f795db13","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:17.932Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1647bbb6-0076-4c69-9b17-dc39f795db13","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:17.933Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1647bbb6-0076-4c69-9b17-dc39f795db13"}
{"level":"info","ts":"2026-06-10T10:35:17.933Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1647bbb6-0076-4c69-9b17-dc39f795db13","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4"}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4"}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:18.134Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"a0bcfc9e-bc40-440f-b63c-a00462437ff4","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:18.381Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154"}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154"}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:18.382Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"fbb986ee-68c9-4d8a-9691-05749f5fc154","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:19.413Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2"}
{"level":"info","ts":"2026-06-10T10:35:19.413Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2"}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:35:19.414Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"e8dc568a-b95a-4979-aad2-e2c0b89225a2","name":"counter-ksbpl","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"94ecb7b4-c65c-4cc9-aa46-a2d8450af267","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"9385ef6a-1f5f-4a27-abb7-5cdcdd8ffccc","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"94ecb7b4-c65c-4cc9-aa46-a2d8450af267","pod":"counter-ksbpl-head-827hc","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"94ecb7b4-c65c-4cc9-aa46-a2d8450af267","pod":"counter-ksbpl-small-group-worker-lh7wz","ip":"10.132.0.46"}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"94ecb7b4-c65c-4cc9-aa46-a2d8450af267","rayCluster":"counter-ksbpl","podIPs":["10.134.0.24","10.132.0.46"]}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8f9f7164-c76a-4f91-a366-a871d45205ab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8f9f7164-c76a-4f91-a366-a871d45205ab","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8f9f7164-c76a-4f91-a366-a871d45205ab","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8f9f7164-c76a-4f91-a366-a871d45205ab"}
{"level":"info","ts":"2026-06-10T10:35:19.460Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8f9f7164-c76a-4f91-a366-a871d45205ab","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:19.528Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"9385ef6a-1f5f-4a27-abb7-5cdcdd8ffccc","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:19.541Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"9385ef6a-1f5f-4a27-abb7-5cdcdd8ffccc","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:19.541Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"9385ef6a-1f5f-4a27-abb7-5cdcdd8ffccc","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:20.933Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"64ef1bad-7776-4026-b647-13d0dc13d203","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:20.933Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"64ef1bad-7776-4026-b647-13d0dc13d203","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:20.933Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"64ef1bad-7776-4026-b647-13d0dc13d203","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:20.933Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"64ef1bad-7776-4026-b647-13d0dc13d203"}
{"level":"info","ts":"2026-06-10T10:35:20.933Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"64ef1bad-7776-4026-b647-13d0dc13d203","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470"}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470"}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","podName":"counter-ksbpl-head-827hc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:21.452Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"3d58b72a-7a32-48a2-a57e-00b4fc539470","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:23.933Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"282918ff-140a-4bb5-b334-c2d1d18accc5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:23.933Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"282918ff-140a-4bb5-b334-c2d1d18accc5","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:23.933Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"282918ff-140a-4bb5-b334-c2d1d18accc5","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:23.934Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"282918ff-140a-4bb5-b334-c2d1d18accc5"}
{"level":"info","ts":"2026-06-10T10:35:23.934Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"282918ff-140a-4bb5-b334-c2d1d18accc5","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:26.934Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"809540fb-4b38-493d-a138-12be9498ed1a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:26.934Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"809540fb-4b38-493d-a138-12be9498ed1a","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:26.934Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"809540fb-4b38-493d-a138-12be9498ed1a","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:26.934Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"809540fb-4b38-493d-a138-12be9498ed1a"}
{"level":"info","ts":"2026-06-10T10:35:26.934Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"809540fb-4b38-493d-a138-12be9498ed1a","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93"}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93"}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:35:27.153Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"5336033a-8e14-4053-908f-55a553d2ae93","name":"counter-ksbpl","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"2ac0f5ce-2c1f-48dd-827d-9466f7797427","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"5753c925-e07f-473d-bc7d-01793a672c55","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d5d761b-c9d8-44b0-8c22-bbe2c4401b70","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d5d761b-c9d8-44b0-8c22-bbe2c4401b70","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"2ac0f5ce-2c1f-48dd-827d-9466f7797427","pod":"counter-ksbpl-head-827hc","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"2ac0f5ce-2c1f-48dd-827d-9466f7797427","pod":"counter-ksbpl-small-group-worker-lh7wz","ip":"10.132.0.46"}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d5d761b-c9d8-44b0-8c22-bbe2c4401b70","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"2ac0f5ce-2c1f-48dd-827d-9466f7797427","rayCluster":"counter-ksbpl","podIPs":["10.134.0.24","10.132.0.46"]}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d5d761b-c9d8-44b0-8c22-bbe2c4401b70"}
{"level":"info","ts":"2026-06-10T10:35:27.166Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d5d761b-c9d8-44b0-8c22-bbe2c4401b70","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:27.176Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"5753c925-e07f-473d-bc7d-01793a672c55","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:27.188Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"5753c925-e07f-473d-bc7d-01793a672c55","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:27.188Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"5753c925-e07f-473d-bc7d-01793a672c55","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7"}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7"}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","podName":"counter-ksbpl-small-group-worker-lh7wz","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:35:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:29.166Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"85bdfe6b-5bb5-4ba5-9095-aaa3d98faaa7","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:29.935Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"9946645c-fdfd-4379-a2bd-6a2d0348b9e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:29.935Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"9946645c-fdfd-4379-a2bd-6a2d0348b9e7","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:29.935Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"9946645c-fdfd-4379-a2bd-6a2d0348b9e7","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:29.935Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"9946645c-fdfd-4379-a2bd-6a2d0348b9e7"}
{"level":"info","ts":"2026-06-10T10:35:29.935Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"9946645c-fdfd-4379-a2bd-6a2d0348b9e7","RayCluster":"counter-ksbpl","State":""}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e"}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e"}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:35:30.425Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"645d7468-0596-4225-9f29-0718f06d451e","name":"counter-ksbpl","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"7d60aca9-82e1-476e-bd19-1792ad48c543","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0bc8f72a-45c3-4f55-b733-9edd7795d746","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"7d60aca9-82e1-476e-bd19-1792ad48c543","pod":"counter-ksbpl-head-827hc","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"7d60aca9-82e1-476e-bd19-1792ad48c543","pod":"counter-ksbpl-small-group-worker-lh7wz","ip":"10.132.0.46"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"7d60aca9-82e1-476e-bd19-1792ad48c543","rayCluster":"counter-ksbpl","podIPs":["10.134.0.24","10.132.0.46"]}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","head service name":"counter-ksbpl-head-svc","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","head service URL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c"}
{"level":"info","ts":"2026-06-10T10:35:30.438Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","command":["if","!","ray","job","status","--address","http://counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","counter-4k889",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-4k889","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","--follow","counter-4k889"]}
{"level":"info","ts":"2026-06-10T10:35:30.448Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-10T10:35:30.449Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","SubmissionMode":"K8sJobMode","RayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.449Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:35:30.449Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"75a9d8c7-2bb0-42c1-b0ec-11e55ca1212c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:35:30.452Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0bc8f72a-45c3-4f55-b733-9edd7795d746","name":"counter-ksbpl-head"}
{"level":"info","ts":"2026-06-10T10:35:30.461Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:30.461Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.461Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:30.461Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3"}
{"level":"info","ts":"2026-06-10T10:35:30.461Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","head service name":"counter-ksbpl-head-svc","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:35:30.461Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","head service URL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:35:30.462Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-10T10:35:30.462Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","SubmissionMode":"K8sJobMode","RayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.462Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:35:30.462Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:35:30.463Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0bc8f72a-45c3-4f55-b733-9edd7795d746","name":"counter-ksbpl-workers"}
{"level":"info","ts":"2026-06-10T10:35:30.463Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"0bc8f72a-45c3-4f55-b733-9edd7795d746","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.467Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-10T10:35:30.467Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3"}
{"level":"error","ts":"2026-06-10T10:35:30.467Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3d24d1ee-5f40-4e0a-9286-0830e8d512a3","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.467Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5ed0eb73-2a7c-436c-b45a-830995607be5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:30.467Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5ed0eb73-2a7c-436c-b45a-830995607be5","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.467Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5ed0eb73-2a7c-436c-b45a-830995607be5","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:30.499Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5ed0eb73-2a7c-436c-b45a-830995607be5"}
{"level":"error","ts":"2026-06-10T10:35:30.510Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5ed0eb73-2a7c-436c-b45a-830995607be5","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.510Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5ed0eb73-2a7c-436c-b45a-830995607be5"}
{"level":"error","ts":"2026-06-10T10:35:30.510Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5ed0eb73-2a7c-436c-b45a-830995607be5","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.510Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b8596e99-8a7e-4079-b708-0249e04b097a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:30.510Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b8596e99-8a7e-4079-b708-0249e04b097a","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.510Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b8596e99-8a7e-4079-b708-0249e04b097a","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:30.510Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b8596e99-8a7e-4079-b708-0249e04b097a"}
{"level":"error","ts":"2026-06-10T10:35:30.515Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b8596e99-8a7e-4079-b708-0249e04b097a","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.515Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b8596e99-8a7e-4079-b708-0249e04b097a"}
{"level":"error","ts":"2026-06-10T10:35:30.515Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b8596e99-8a7e-4079-b708-0249e04b097a","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.520Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cd6acbef-fba9-4a8c-9bae-04a36f37de78","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:30.520Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cd6acbef-fba9-4a8c-9bae-04a36f37de78","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.520Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cd6acbef-fba9-4a8c-9bae-04a36f37de78","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:30.520Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cd6acbef-fba9-4a8c-9bae-04a36f37de78"}
{"level":"error","ts":"2026-06-10T10:35:30.524Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cd6acbef-fba9-4a8c-9bae-04a36f37de78","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.524Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cd6acbef-fba9-4a8c-9bae-04a36f37de78"}
{"level":"error","ts":"2026-06-10T10:35:30.524Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cd6acbef-fba9-4a8c-9bae-04a36f37de78","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.565Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5807e0a5-0aaf-416d-a842-2b9bc844ad41","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:30.565Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5807e0a5-0aaf-416d-a842-2b9bc844ad41","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.565Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5807e0a5-0aaf-416d-a842-2b9bc844ad41","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:30.565Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5807e0a5-0aaf-416d-a842-2b9bc844ad41"}
{"level":"error","ts":"2026-06-10T10:35:30.570Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5807e0a5-0aaf-416d-a842-2b9bc844ad41","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.570Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5807e0a5-0aaf-416d-a842-2b9bc844ad41"}
{"level":"error","ts":"2026-06-10T10:35:30.570Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5807e0a5-0aaf-416d-a842-2b9bc844ad41","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.650Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"801015d0-3d66-4111-a6bd-48d0878b10c3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:30.650Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"801015d0-3d66-4111-a6bd-48d0878b10c3","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.650Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"801015d0-3d66-4111-a6bd-48d0878b10c3","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:30.650Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"801015d0-3d66-4111-a6bd-48d0878b10c3"}
{"level":"error","ts":"2026-06-10T10:35:30.656Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"801015d0-3d66-4111-a6bd-48d0878b10c3","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.656Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"801015d0-3d66-4111-a6bd-48d0878b10c3"}
{"level":"error","ts":"2026-06-10T10:35:30.656Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"801015d0-3d66-4111-a6bd-48d0878b10c3","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.816Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"695272b9-ad8f-43f3-833d-2201c851b499","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:30.816Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"695272b9-ad8f-43f3-833d-2201c851b499","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:30.816Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"695272b9-ad8f-43f3-833d-2201c851b499","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:30.816Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"695272b9-ad8f-43f3-833d-2201c851b499"}
{"level":"error","ts":"2026-06-10T10:35:30.822Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"695272b9-ad8f-43f3-833d-2201c851b499","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:30.822Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"695272b9-ad8f-43f3-833d-2201c851b499"}
{"level":"error","ts":"2026-06-10T10:35:30.822Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"695272b9-ad8f-43f3-833d-2201c851b499","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:31.143Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"c51f4faf-f2fe-4150-8221-1505a4e57077","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:31.143Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"c51f4faf-f2fe-4150-8221-1505a4e57077","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:31.143Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"c51f4faf-f2fe-4150-8221-1505a4e57077","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:31.143Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"c51f4faf-f2fe-4150-8221-1505a4e57077"}
{"level":"error","ts":"2026-06-10T10:35:31.148Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"c51f4faf-f2fe-4150-8221-1505a4e57077","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:31.148Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"c51f4faf-f2fe-4150-8221-1505a4e57077"}
{"level":"error","ts":"2026-06-10T10:35:31.148Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"c51f4faf-f2fe-4150-8221-1505a4e57077","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:31.468Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e0aadf38-95bf-4db1-99a0-b8afe4e627c6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:31.468Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e0aadf38-95bf-4db1-99a0-b8afe4e627c6","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:31.468Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e0aadf38-95bf-4db1-99a0-b8afe4e627c6","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:31.468Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e0aadf38-95bf-4db1-99a0-b8afe4e627c6"}
{"level":"error","ts":"2026-06-10T10:35:31.473Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e0aadf38-95bf-4db1-99a0-b8afe4e627c6","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:31.473Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e0aadf38-95bf-4db1-99a0-b8afe4e627c6"}
{"level":"error","ts":"2026-06-10T10:35:31.473Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"e0aadf38-95bf-4db1-99a0-b8afe4e627c6","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:31.789Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4442506a-bc43-418e-b465-c4026ad7d3d2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:31.789Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4442506a-bc43-418e-b465-c4026ad7d3d2","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:31.789Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4442506a-bc43-418e-b465-c4026ad7d3d2","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:31.789Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4442506a-bc43-418e-b465-c4026ad7d3d2"}
{"level":"error","ts":"2026-06-10T10:35:31.795Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4442506a-bc43-418e-b465-c4026ad7d3d2","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:31.795Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4442506a-bc43-418e-b465-c4026ad7d3d2"}
{"level":"error","ts":"2026-06-10T10:35:31.795Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4442506a-bc43-418e-b465-c4026ad7d3d2","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd"}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd"}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","1 head service found":"counter-ksbpl-head-svc"}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","Found 1 head Pod":"counter-ksbpl-head-827hc","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","head Pod":"counter-ksbpl-head-827hc","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-ksbpl-head-827hc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","worker Pod":"counter-ksbpl-small-group-worker-lh7wz","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-ksbpl-small-group-worker-lh7wz. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:35:32.438Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"6b4ad365-0bc2-413f-9617-967e9c1487fd","seconds":300}
{"level":"info","ts":"2026-06-10T10:35:34.356Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"11d564cf-3c6d-4e6f-b68e-9650358a88f3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:34.356Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"11d564cf-3c6d-4e6f-b68e-9650358a88f3","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:34.356Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"11d564cf-3c6d-4e6f-b68e-9650358a88f3","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:34.356Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"11d564cf-3c6d-4e6f-b68e-9650358a88f3"}
{"level":"error","ts":"2026-06-10T10:35:34.361Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"11d564cf-3c6d-4e6f-b68e-9650358a88f3","JobId":"counter-4k889","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:34.361Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"11d564cf-3c6d-4e6f-b68e-9650358a88f3"}
{"level":"error","ts":"2026-06-10T10:35:34.361Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"11d564cf-3c6d-4e6f-b68e-9650358a88f3","error":"Job counter-4k889 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:36.211Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"a342f54c-0076-4e03-88cb-4fb521b8b171","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:36.211Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"a342f54c-0076-4e03-88cb-4fb521b8b171","pod":"counter-ksbpl-head-827hc","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-10T10:35:36.211Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"a342f54c-0076-4e03-88cb-4fb521b8b171","pod":"counter-ksbpl-small-group-worker-lh7wz","ip":"10.132.0.46"}
{"level":"info","ts":"2026-06-10T10:35:36.211Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"a342f54c-0076-4e03-88cb-4fb521b8b171","rayCluster":"counter-ksbpl","podIPs":["10.134.0.24","10.132.0.46"]}
{"level":"info","ts":"2026-06-10T10:35:39.482Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4e214fb8-79cd-48a6-a485-c129b76eacb9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:39.482Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4e214fb8-79cd-48a6-a485-c129b76eacb9","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:39.482Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4e214fb8-79cd-48a6-a485-c129b76eacb9","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:39.482Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4e214fb8-79cd-48a6-a485-c129b76eacb9"}
{"level":"info","ts":"2026-06-10T10:35:39.489Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4e214fb8-79cd-48a6-a485-c129b76eacb9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:39.489Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"4e214fb8-79cd-48a6-a485-c129b76eacb9","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:35:39.502Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d2731b9-2955-46e1-9fe9-a77d3cc9de39","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:39.502Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d2731b9-2955-46e1-9fe9-a77d3cc9de39","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:39.502Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d2731b9-2955-46e1-9fe9-a77d3cc9de39","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:39.503Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d2731b9-2955-46e1-9fe9-a77d3cc9de39"}
{"level":"info","ts":"2026-06-10T10:35:39.506Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"8d2731b9-2955-46e1-9fe9-a77d3cc9de39","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:42.502Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3ef214ab-1064-42f4-98ff-42143d4ac077","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:42.502Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3ef214ab-1064-42f4-98ff-42143d4ac077","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:42.502Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3ef214ab-1064-42f4-98ff-42143d4ac077","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:42.503Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3ef214ab-1064-42f4-98ff-42143d4ac077"}
{"level":"info","ts":"2026-06-10T10:35:42.506Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3ef214ab-1064-42f4-98ff-42143d4ac077","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:42.506Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"3ef214ab-1064-42f4-98ff-42143d4ac077","old JobStatus":"PENDING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:35:42.522Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"ee061fd2-e7ea-4fa9-a31f-05d91772fc38","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:42.522Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"ee061fd2-e7ea-4fa9-a31f-05d91772fc38","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:42.522Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"ee061fd2-e7ea-4fa9-a31f-05d91772fc38","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:42.522Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"ee061fd2-e7ea-4fa9-a31f-05d91772fc38"}
{"level":"info","ts":"2026-06-10T10:35:42.525Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"ee061fd2-e7ea-4fa9-a31f-05d91772fc38","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:45.521Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b1f8090f-7917-4141-af31-3c728a680244","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:45.521Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b1f8090f-7917-4141-af31-3c728a680244","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:45.521Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b1f8090f-7917-4141-af31-3c728a680244","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:45.521Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b1f8090f-7917-4141-af31-3c728a680244"}
{"level":"info","ts":"2026-06-10T10:35:45.525Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b1f8090f-7917-4141-af31-3c728a680244","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:46.912Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7ebf470c-53b0-4d8c-833a-b300b39786f9","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:35:46.913Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7ebf470c-53b0-4d8c-833a-b300b39786f9","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:35:46.913Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7ebf470c-53b0-4d8c-833a-b300b39786f9","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:35:46.913Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7ebf470c-53b0-4d8c-833a-b300b39786f9","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:35:48.525Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5aa1a973-d41c-455f-ad61-55da30709d0f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:48.525Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5aa1a973-d41c-455f-ad61-55da30709d0f","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:48.525Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5aa1a973-d41c-455f-ad61-55da30709d0f","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:48.525Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5aa1a973-d41c-455f-ad61-55da30709d0f"}
{"level":"info","ts":"2026-06-10T10:35:48.529Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"5aa1a973-d41c-455f-ad61-55da30709d0f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:49.244Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d661c413-b332-4e6c-b0d3-7fe461d37d0e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:49.244Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d661c413-b332-4e6c-b0d3-7fe461d37d0e","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:49.244Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d661c413-b332-4e6c-b0d3-7fe461d37d0e","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:49.244Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d661c413-b332-4e6c-b0d3-7fe461d37d0e"}
{"level":"info","ts":"2026-06-10T10:35:49.299Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d661c413-b332-4e6c-b0d3-7fe461d37d0e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:50.251Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"65f72f34-89cb-4111-979d-92cf59084153","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:50.251Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"65f72f34-89cb-4111-979d-92cf59084153","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.252Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"65f72f34-89cb-4111-979d-92cf59084153","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:50.252Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"65f72f34-89cb-4111-979d-92cf59084153"}
{"level":"info","ts":"2026-06-10T10:35:50.255Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"65f72f34-89cb-4111-979d-92cf59084153","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:50.270Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"43fee7e6-7e8f-455e-8e83-99872efee09c","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:50.270Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"43fee7e6-7e8f-455e-8e83-99872efee09c","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.270Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"43fee7e6-7e8f-455e-8e83-99872efee09c","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:50.270Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"43fee7e6-7e8f-455e-8e83-99872efee09c"}
{"level":"info","ts":"2026-06-10T10:35:50.273Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"43fee7e6-7e8f-455e-8e83-99872efee09c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:35:38Z","endTime":"2026-06-10T10:35:41Z"},"jobId":"counter-4k889","rayClusterName":"counter-ksbpl","dashboardURL":"counter-ksbpl-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-10T10:35:05Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:35:30Z","stateTransitionTimes":{"ready":"2026-06-10T10:35:30Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.24","serviceIP":"10.134.0.24","podName":"counter-ksbpl-head-827hc","serviceName":"counter-ksbpl-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:35:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:35:30Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:35:06Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:35:06Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:35:50.273Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"43fee7e6-7e8f-455e-8e83-99872efee09c","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-10T10:35:50.286Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1fa6556c-f8a6-40b4-9409-1d3a707ce7c2","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:50.286Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1fa6556c-f8a6-40b4-9409-1d3a707ce7c2","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:35:50 +0000 UTC","Now":"2026-06-10T10:35:50.286Z","ShutdownTime":"2026-06-10T10:35:50.000Z"}
{"level":"info","ts":"2026-06-10T10:35:50.286Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1fa6556c-f8a6-40b4-9409-1d3a707ce7c2","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1fa6556c-f8a6-40b4-9409-1d3a707ce7c2","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1fa6556c-f8a6-40b4-9409-1d3a707ce7c2","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"1fa6556c-f8a6-40b4-9409-1d3a707ce7c2","RayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"2045e1fb-bc0e-42d3-96cb-9d73c55c8732","rayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"reconcileID":"2bc70408-0e5f-4146-b967-eca023eb2735"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"427b4b3d-0c17-43a9-a2a7-28036c40ab32"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d26b11b6-ee83-4a48-b1d7-d114990f5726","namespacedName":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d26b11b6-ee83-4a48-b1d7-d114990f5726","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d26b11b6-ee83-4a48-b1d7-d114990f5726","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"2045e1fb-bc0e-42d3-96cb-9d73c55c8732","pod":"counter-ksbpl-head-827hc","ip":"10.134.0.24"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"2045e1fb-bc0e-42d3-96cb-9d73c55c8732","pod":"counter-ksbpl-small-group-worker-lh7wz","ip":"10.132.0.46"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f73c2edf-1009-423b-887b-f6c665716028","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f73c2edf-1009-423b-887b-f6c665716028","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:35:50 +0000 UTC","Now":"2026-06-10T10:35:50.295Z","ShutdownTime":"2026-06-10T10:35:50.000Z"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f73c2edf-1009-423b-887b-f6c665716028","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f73c2edf-1009-423b-887b-f6c665716028","RayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f73c2edf-1009-423b-887b-f6c665716028","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f73c2edf-1009-423b-887b-f6c665716028","RayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"2045e1fb-bc0e-42d3-96cb-9d73c55c8732","rayCluster":"counter-ksbpl","podIPs":["10.134.0.24","10.132.0.46"]}
{"level":"info","ts":"2026-06-10T10:35:50.295Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d26b11b6-ee83-4a48-b1d7-d114990f5726","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:35:50.302Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d26b11b6-ee83-4a48-b1d7-d114990f5726","name":"kuberay-gateway-access","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:35:50.306Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d26b11b6-ee83-4a48-b1d7-d114990f5726","name":"test-ns-k92wt-counter-ksbpl","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:35:50.311Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d26b11b6-ee83-4a48-b1d7-d114990f5726","configMap":"kube-rbac-proxy-config-counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.315Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d26b11b6-ee83-4a48-b1d7-d114990f5726","serviceAccount":"counter-ksbpl-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:35:50.323Z","logger":"raycluster-resource","msg":"default","name":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.323Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"counter-ksbpl","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:35:50.335Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"d26b11b6-ee83-4a48-b1d7-d114990f5726","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.335Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fe567ea5-e1ae-47b2-9c02-8e5ae3405256","namespacedName":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:50.335Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fe567ea5-e1ae-47b2-9c02-8e5ae3405256","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.335Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fe567ea5-e1ae-47b2-9c02-8e5ae3405256","cluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.335Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fe567ea5-e1ae-47b2-9c02-8e5ae3405256","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:35:50.335Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fe567ea5-e1ae-47b2-9c02-8e5ae3405256","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:35:50.336Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"30f9c194-ee20-4557-b345-cd5528431fde"}
{"level":"info","ts":"2026-06-10T10:35:50.336Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"65e5aa3f-9c88-4c3f-8de1-ddb56fbe097b","namespace":"test-ns-k92wt","clusterName":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:50.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d4c432fd-0fcd-4bb2-a3af-e2c0ebab146a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:50.336Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d4c432fd-0fcd-4bb2-a3af-e2c0ebab146a","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:35:50 +0000 UTC","Now":"2026-06-10T10:35:50.336Z","ShutdownTime":"2026-06-10T10:35:50.000Z"}
{"level":"info","ts":"2026-06-10T10:35:50.336Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d4c432fd-0fcd-4bb2-a3af-e2c0ebab146a","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:35:50.336Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d4c432fd-0fcd-4bb2-a3af-e2c0ebab146a","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:50.336Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d4c432fd-0fcd-4bb2-a3af-e2c0ebab146a","isClusterDeleted":true}
{"level":"info","ts":"2026-06-10T10:35:50.336Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"d4c432fd-0fcd-4bb2-a3af-e2c0ebab146a","RayCluster":"counter-ksbpl"}
{"level":"error","ts":"2026-06-10T10:35:50.339Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fe567ea5-e1ae-47b2-9c02-8e5ae3405256","error":"rayclusters.ray.io \"counter-ksbpl\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:50.339Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fe567ea5-e1ae-47b2-9c02-8e5ae3405256"}
{"level":"error","ts":"2026-06-10T10:35:50.339Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"fe567ea5-e1ae-47b2-9c02-8e5ae3405256","error":"rayclusters.ray.io \"counter-ksbpl\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:35:50.339Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8c67d76d-bc67-483f-a976-b3b0e9ee162a","namespacedName":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:50.339Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8c67d76d-bc67-483f-a976-b3b0e9ee162a"}
{"level":"info","ts":"2026-06-10T10:35:50.339Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8c67d76d-bc67-483f-a976-b3b0e9ee162a","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:35:50.339Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"8c67d76d-bc67-483f-a976-b3b0e9ee162a","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:35:50.345Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f4e0bb16-844c-4ead-94f8-44574a710c9c","namespacedName":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:50.345Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f4e0bb16-844c-4ead-94f8-44574a710c9c"}
{"level":"info","ts":"2026-06-10T10:35:50.345Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f4e0bb16-844c-4ead-94f8-44574a710c9c","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:35:50.345Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"f4e0bb16-844c-4ead-94f8-44574a710c9c","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:35:50.357Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"cd91bfb7-4eb0-46f8-9f08-b31213c28520"}
{"level":"info","ts":"2026-06-10T10:35:50.399Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"e1956ae0-d3da-4a84-ae7c-9be9814c4a1a","namespacedName":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:50.399Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"e1956ae0-d3da-4a84-ae7c-9be9814c4a1a"}
{"level":"info","ts":"2026-06-10T10:35:50.399Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"e1956ae0-d3da-4a84-ae7c-9be9814c4a1a","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:35:50.399Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"e1956ae0-d3da-4a84-ae7c-9be9814c4a1a","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:35:50.399Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"2f74bd70-7604-4756-96cd-eb244f106247"}
{"level":"info","ts":"2026-06-10T10:35:50.409Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"65e5aa3f-9c88-4c3f-8de1-ddb56fbe097b","namespace":"test-ns-k92wt","clusterName":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:51.177Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cca150a7-d6cd-4da5-aae2-98aca2113163","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:51.177Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cca150a7-d6cd-4da5-aae2-98aca2113163","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:35:50 +0000 UTC","Now":"2026-06-10T10:35:51.177Z","ShutdownTime":"2026-06-10T10:35:50.000Z"}
{"level":"info","ts":"2026-06-10T10:35:51.177Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cca150a7-d6cd-4da5-aae2-98aca2113163","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:35:51.177Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cca150a7-d6cd-4da5-aae2-98aca2113163","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:51.177Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cca150a7-d6cd-4da5-aae2-98aca2113163","isClusterDeleted":true}
{"level":"info","ts":"2026-06-10T10:35:51.177Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"cca150a7-d6cd-4da5-aae2-98aca2113163","RayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:35:51.529Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f7ac017e-ed25-4dda-841f-00705fe1e6db","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:35:51.529Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f7ac017e-ed25-4dda-841f-00705fe1e6db","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:35:50 +0000 UTC","Now":"2026-06-10T10:35:51.529Z","ShutdownTime":"2026-06-10T10:35:50.000Z"}
{"level":"info","ts":"2026-06-10T10:35:51.529Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f7ac017e-ed25-4dda-841f-00705fe1e6db","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:35:51.529Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f7ac017e-ed25-4dda-841f-00705fe1e6db","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:35:51.529Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f7ac017e-ed25-4dda-841f-00705fe1e6db","isClusterDeleted":true}
{"level":"info","ts":"2026-06-10T10:35:51.529Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"f7ac017e-ed25-4dda-841f-00705fe1e6db","RayCluster":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:36:21.189Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"13272afb-d517-4aa0-8047-7ef37040e24a","DeletionTimestamp":"2026-06-10 10:36:21 +0000 UTC"}
{"level":"info","ts":"2026-06-10T10:36:21.189Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"13272afb-d517-4aa0-8047-7ef37040e24a","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:36:21.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"74a1531f-cd26-46cf-9415-be263895a4b0","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:21.223Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"74a1531f-cd26-46cf-9415-be263895a4b0","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:36:21.241Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"74a1531f-cd26-46cf-9415-be263895a4b0"}
{"level":"info","ts":"2026-06-10T10:36:21.241Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"74a1531f-cd26-46cf-9415-be263895a4b0","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:36:21.241Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"74a1531f-cd26-46cf-9415-be263895a4b0","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:36:21.241Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"74a1531f-cd26-46cf-9415-be263895a4b0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:36:21.266Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"b5634c7c-aba0-46a2-8ca6-64d77f93052d"}
{"level":"info","ts":"2026-06-10T10:36:21.266Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"654d1322-488c-453f-a93a-ec7962eefb6c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:21.266Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"654d1322-488c-453f-a93a-ec7962eefb6c","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.266Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"654d1322-488c-453f-a93a-ec7962eefb6c","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.266Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"654d1322-488c-453f-a93a-ec7962eefb6c","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.272Z","logger":"raycluster-resource","msg":"default","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.272Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-9zlsp","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:36:21.300Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7000c415-0396-45a8-bc6a-f1dc7fb63dfd","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.300Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"f6158324-8e1b-4e04-ac07-d5add99107ff","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.300Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba"}
{"level":"info","ts":"2026-06-10T10:36:21.300Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba"}
{"level":"info","ts":"2026-06-10T10:36:21.300Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7000c415-0396-45a8-bc6a-f1dc7fb63dfd","mode":"IntegratedOAuth","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.300Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7000c415-0396-45a8-bc6a-f1dc7fb63dfd","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.300Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7000c415-0396-45a8-bc6a-f1dc7fb63dfd","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.300Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.300Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.306Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"654d1322-488c-453f-a93a-ec7962eefb6c","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"654d1322-488c-453f-a93a-ec7962eefb6c"}
{"level":"info","ts":"2026-06-10T10:36:21.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"654d1322-488c-453f-a93a-ec7962eefb6c","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:21.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"78d1cf97-0453-4fa1-9fb0-ec1cce2c697c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:21.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"78d1cf97-0453-4fa1-9fb0-ec1cce2c697c","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"78d1cf97-0453-4fa1-9fb0-ec1cce2c697c","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"78d1cf97-0453-4fa1-9fb0-ec1cce2c697c"}
{"level":"info","ts":"2026-06-10T10:36:21.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"78d1cf97-0453-4fa1-9fb0-ec1cce2c697c","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:21.311Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba","name":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:21.311Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba","error":"head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"error","ts":"2026-06-10T10:36:21.311Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.311Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"info","ts":"2026-06-10T10:36:21.311Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba","serviceSelectors":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/identifier":"fail-9zlsp-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba","error":"unable to find head service. cluster name fail-9zlsp, filter labels [test-ns-k92wt map[ray.io/cluster:fail-9zlsp ray.io/identifier:fail-9zlsp-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba"}
{"level":"error","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f12f57b5-4296-48ca-8a1e-1605d1f48fba","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa"}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa"}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","error":"head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"error","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.312Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.313Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:36:21.313Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","name":"fail-9zlsp","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:21.317Z","logger":"raycluster-resource","msg":"default","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.317Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-9zlsp","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:36:21.325Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"f6158324-8e1b-4e04-ac07-d5add99107ff","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.326Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"fec20175-68db-440b-aefd-ee7262473d8d","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.332Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.333Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.335Z","logger":"raycluster-resource","msg":"default","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.335Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-9zlsp","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:36:21.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"55945c94-f01d-48f7-bfb8-ba0f93e699a4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:21.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"55945c94-f01d-48f7-bfb8-ba0f93e699a4","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"55945c94-f01d-48f7-bfb8-ba0f93e699a4","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"55945c94-f01d-48f7-bfb8-ba0f93e699a4"}
{"level":"info","ts":"2026-06-10T10:36:21.336Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"55945c94-f01d-48f7-bfb8-ba0f93e699a4","RayCluster":"fail-9zlsp","State":""}
{"level":"error","ts":"2026-06-10T10:36:21.341Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7000c415-0396-45a8-bc6a-f1dc7fb63dfd","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.341Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7000c415-0396-45a8-bc6a-f1dc7fb63dfd"}
{"level":"error","ts":"2026-06-10T10:36:21.341Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7000c415-0396-45a8-bc6a-f1dc7fb63dfd","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.341Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.341Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","mode":"IntegratedOAuth","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.341Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.341Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.341Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"fec20175-68db-440b-aefd-ee7262473d8d","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa"}
{"level":"error","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"4e18e3b2-f663-4139-b83d-45074d32baaa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1","error":"head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"error","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa","rayCluster":"fail-9zlsp","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1"}
{"level":"error","ts":"2026-06-10T10:36:21.399Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f7fbfe55-4139-4ea3-8b5a-8a807bcd0bd1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"raycluster-resource","msg":"default","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.399Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-9zlsp","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:36:21.403Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"fec20175-68db-440b-aefd-ee7262473d8d","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:21.403Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"fec20175-68db-440b-aefd-ee7262473d8d","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.403Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a3f5555d-d430-451d-acf4-0765faed7d19","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.411Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f6f10490-4e64-41ed-9a48-b7771dfc6951","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:21.411Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f6f10490-4e64-41ed-9a48-b7771dfc6951","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.411Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f6f10490-4e64-41ed-9a48-b7771dfc6951","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.411Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f6f10490-4e64-41ed-9a48-b7771dfc6951"}
{"level":"info","ts":"2026-06-10T10:36:21.411Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f6f10490-4e64-41ed-9a48-b7771dfc6951","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:21.415Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.416Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a3f5555d-d430-451d-acf4-0765faed7d19","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:21.416Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","name":"fail-9zlsp-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9"}
{"level":"info","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9"}
{"level":"info","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9","error":"head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"error","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"info","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9"}
{"level":"error","ts":"2026-06-10T10:36:21.420Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"03a7e2c0-a863-4fa2-ad7f-f8e327fad4f9","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0"}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0"}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa"}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0","error":"head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"error","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0"}
{"level":"error","ts":"2026-06-10T10:36:21.499Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8a00133b-f4e8-4af6-8387-59793a0872f0","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.499Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","name":"kuberay-gateway-access","namespace":"test-ns-k92wt","operation":"created"}
{"level":"info","ts":"2026-06-10T10:36:21.500Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa","rayCluster":"fail-9zlsp","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:36:21.504Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a3f5555d-d430-451d-acf4-0765faed7d19","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:21.504Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a3f5555d-d430-451d-acf4-0765faed7d19","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.504Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a2a0c339-b10b-47dd-a65a-f21d27051f46","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.508Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","name":"test-ns-k92wt-fail-9zlsp","operation":"created"}
{"level":"info","ts":"2026-06-10T10:36:21.513Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","name":"kube-rbac-proxy-config-fail-9zlsp","operation":"created"}
{"level":"info","ts":"2026-06-10T10:36:21.521Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a2a0c339-b10b-47dd-a65a-f21d27051f46","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:21.521Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa"}
{"level":"info","ts":"2026-06-10T10:36:21.521Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa","rayCluster":"fail-9zlsp","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:36:21.521Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c349a49f-404c-405d-a65b-d6fc8f0c36fa"}
{"level":"info","ts":"2026-06-10T10:36:21.521Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"01a377f3-7c49-44d1-b4ce-a0c4d4b2edd3","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.521Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"01a377f3-7c49-44d1-b4ce-a0c4d4b2edd3"}
{"level":"error","ts":"2026-06-10T10:36:21.599Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.599Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a"}
{"level":"error","ts":"2026-06-10T10:36:21.599Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"7cfc2bb3-48ba-4784-a43a-deb873e1128a","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.599Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a2a0c339-b10b-47dd-a65a-f21d27051f46","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:21.599Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a2a0c339-b10b-47dd-a65a-f21d27051f46","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.599Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"bb692f41-1b80-4d9b-809c-0393eaf5810f","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.599Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"bb692f41-1b80-4d9b-809c-0393eaf5810f","mode":"IntegratedOAuth","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.599Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"bb692f41-1b80-4d9b-809c-0393eaf5810f","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.599Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5"}
{"level":"info","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5"}
{"level":"info","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5","error":"head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"error","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"info","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5"}
{"level":"error","ts":"2026-06-10T10:36:21.600Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"47c6970f-a553-4eab-90bd-a4981a144bf5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"bb692f41-1b80-4d9b-809c-0393eaf5810f","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"e6d8dcd7-d820-41f0-b638-5435b08f9c28","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"e6d8dcd7-d820-41f0-b638-5435b08f9c28","mode":"IntegratedOAuth","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"e6d8dcd7-d820-41f0-b638-5435b08f9c28","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"0e5b5435-a4df-4adb-a99b-5e77f42bcdcf","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"0e5b5435-a4df-4adb-a99b-5e77f42bcdcf"}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"82470f9e-a13a-4d64-9930-af6d4e22889b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"82470f9e-a13a-4d64-9930-af6d4e22889b","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"82470f9e-a13a-4d64-9930-af6d4e22889b","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.613Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"b0fa5b67-4a85-4463-bce0-8845e44307ce","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.614Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"82470f9e-a13a-4d64-9930-af6d4e22889b"}
{"level":"info","ts":"2026-06-10T10:36:21.614Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"82470f9e-a13a-4d64-9930-af6d4e22889b","RayCluster":"fail-9zlsp","State":""}
{"level":"error","ts":"2026-06-10T10:36:21.621Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"e6d8dcd7-d820-41f0-b638-5435b08f9c28","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.621Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"e6d8dcd7-d820-41f0-b638-5435b08f9c28"}
{"level":"error","ts":"2026-06-10T10:36:21.621Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"e6d8dcd7-d820-41f0-b638-5435b08f9c28","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.624Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"b0fa5b67-4a85-4463-bce0-8845e44307ce","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:21.627Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"bbdbce8d-4b00-474b-9fbd-510d711eb72d","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:21.627Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"bbdbce8d-4b00-474b-9fbd-510d711eb72d","mode":"IntegratedOAuth","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.627Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"bbdbce8d-4b00-474b-9fbd-510d711eb72d","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.635Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"b0fa5b67-4a85-4463-bce0-8845e44307ce","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:21.635Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"b0fa5b67-4a85-4463-bce0-8845e44307ce","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.636Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"bbdbce8d-4b00-474b-9fbd-510d711eb72d","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46"}
{"level":"info","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46"}
{"level":"info","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46","error":"head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"error","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"info","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46"}
{"level":"error","ts":"2026-06-10T10:36:21.761Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"422cb877-5c71-4d24-ac78-57926feb5e46","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee"}
{"level":"info","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee"}
{"level":"info","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee","error":"head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"error","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"info","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee"}
{"level":"error","ts":"2026-06-10T10:36:22.082Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6f54462e-24f6-4484-98d6-728d484af6ee","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e"}
{"level":"info","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e"}
{"level":"info","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e","error":"head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"error","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found"}
{"level":"info","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-9zlsp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e"}
{"level":"error","ts":"2026-06-10T10:36:22.723Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"fb204029-4f17-43cb-8c1b-d5d264b0f15e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-9zlsp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4"}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4"}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4"}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4"}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","mode":"IntegratedOAuth","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","cluster":"fail-9zlsp","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","cluster":"fail-9zlsp","authType":"IntegratedOAuth","serviceAccount":"fail-9zlsp-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-9zlsp","ray.io/group":"headgroup","ray.io/identifier":"fail-9zlsp-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:36:24.004Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","enabled":true}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","name":"fail-9zlsp-head-hvld7"}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4"}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","nodeType":"worker","rayStartParams":{"address":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","rayStartCmd":"ray start  --address=fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:36:24.021Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","enabled":true}
{"level":"info","ts":"2026-06-10T10:36:24.037Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","name":"fail-9zlsp-small-group-worker-zcgkr"}
{"level":"info","ts":"2026-06-10T10:36:24.037Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.037Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.037Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-9zlsp-head-svc"},"newHead":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"}}
{"level":"info","ts":"2026-06-10T10:36:24.037Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"a9eae05b-466f-4c08-be64-b65bd8fdb1f4","name":"fail-9zlsp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:24.051Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-9zlsp-head-svc"},"newHead":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"}}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c7ff3871-b9cc-4a01-8d8d-4f16d630c318","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","name":"fail-9zlsp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"fd82df9e-f0de-42e3-9663-f198df51e131","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"c7ff3871-b9cc-4a01-8d8d-4f16d630c318","rayCluster":"fail-9zlsp","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bcb73eaf-d2ab-4bd5-aca0-e05e94ccbea7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bcb73eaf-d2ab-4bd5-aca0-e05e94ccbea7","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.052Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bcb73eaf-d2ab-4bd5-aca0-e05e94ccbea7","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:24.053Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bcb73eaf-d2ab-4bd5-aca0-e05e94ccbea7"}
{"level":"info","ts":"2026-06-10T10:36:24.053Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bcb73eaf-d2ab-4bd5-aca0-e05e94ccbea7","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:24.099Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","name":"fail-9zlsp","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"fail-9zlsp"}}
{"level":"info","ts":"2026-06-10T10:36:24.110Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"fd82df9e-f0de-42e3-9663-f198df51e131","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa"}
{"level":"error","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"d7ad9ba2-c979-42c1-b4f3-a475a36a22aa","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b"}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b"}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.199Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.200Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:36:24.200Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c45f5e3d-7f7c-451c-aef0-76e280346e8b","name":"fail-9zlsp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:24.299Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"counter","namespace":"test-ns-k92wt"},"reconcileID":"6a60d69b-b695-4877-bb64-04d142d38e4c"}
{"level":"info","ts":"2026-06-10T10:36:24.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"926d08aa-e615-4987-81d9-87679c2679d2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:24.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"926d08aa-e615-4987-81d9-87679c2679d2","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.299Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"fd82df9e-f0de-42e3-9663-f198df51e131","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:24.299Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"fd82df9e-f0de-42e3-9663-f198df51e131","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"926d08aa-e615-4987-81d9-87679c2679d2","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"926d08aa-e615-4987-81d9-87679c2679d2"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"926d08aa-e615-4987-81d9-87679c2679d2","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"37cef61e-6dc9-4378-8b99-154bafec3974","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1b75b7af-2a32-4496-a400-fbdf9f3712d3","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"37cef61e-6dc9-4378-8b99-154bafec3974","rayCluster":"fail-9zlsp","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"04aa36de-f4ef-4150-a927-8b65a52e96f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"04aa36de-f4ef-4150-a927-8b65a52e96f5","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"04aa36de-f4ef-4150-a927-8b65a52e96f5","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"04aa36de-f4ef-4150-a927-8b65a52e96f5"}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"04aa36de-f4ef-4150-a927-8b65a52e96f5","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:24.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.301Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:36:24.301Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","name":"fail-9zlsp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:24.307Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","name":"fail-9zlsp","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"fail-9zlsp"}}
{"level":"info","ts":"2026-06-10T10:36:24.307Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335"}
{"level":"error","ts":"2026-06-10T10:36:24.307Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"ffce0634-3667-4fc2-8df3-7b0d84125335","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-9zlsp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:24.309Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1b75b7af-2a32-4496-a400-fbdf9f3712d3","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a"}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a"}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:24.399Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"44185e66-a5cb-4ee2-9471-12b69d2a063a","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:24.403Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1b75b7af-2a32-4496-a400-fbdf9f3712d3","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:24.403Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1b75b7af-2a32-4496-a400-fbdf9f3712d3","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2"}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2"}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:24.470Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1760fee9-60ff-4dc8-8763-cc84e6b733a2","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5"}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5"}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.499Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.500Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:24.500Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"b5237ffa-6946-4627-9c9d-a8711e8fffc5","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:24.753Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c"}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c"}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:24.754Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"f97bacc7-a956-449a-beb8-a87cf2688d5c","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1"}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1"}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"newHead":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"}}
{"level":"info","ts":"2026-06-10T10:36:25.376Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"41488ad8-1a23-4048-ad49-49db1a8fc0f1","name":"fail-9zlsp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:25.388Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"092191ba-f0b9-42c0-9647-86831ac8254c","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:25.388Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"6b4f561b-f633-40c4-a313-650dedf53d6d","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:25.388Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"092191ba-f0b9-42c0-9647-86831ac8254c","pod":"fail-9zlsp-head-hvld7","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-10T10:36:25.388Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"092191ba-f0b9-42c0-9647-86831ac8254c","pod":"fail-9zlsp-small-group-worker-zcgkr","ip":"10.132.0.47"}
{"level":"info","ts":"2026-06-10T10:36:25.388Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2babafc3-0328-47c2-bbba-22cc1dbc89cf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:25.388Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2babafc3-0328-47c2-bbba-22cc1dbc89cf","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:25.388Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2babafc3-0328-47c2-bbba-22cc1dbc89cf","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:25.388Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2babafc3-0328-47c2-bbba-22cc1dbc89cf"}
{"level":"info","ts":"2026-06-10T10:36:25.388Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2babafc3-0328-47c2-bbba-22cc1dbc89cf","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:25.399Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"6b4f561b-f633-40c4-a313-650dedf53d6d","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:25.399Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"092191ba-f0b9-42c0-9647-86831ac8254c","certificate":"ray-head-cert-fail-9zlsp","dnsNames":["fail-9zlsp-head-svc","localhost","fail-9zlsp-head-svc.test-ns-k92wt.svc","fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.26","10.132.0.47","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:36:25.409Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"092191ba-f0b9-42c0-9647-86831ac8254c","certificate":"ray-worker-cert-fail-9zlsp","dnsNames":["fail-9zlsp-worker-svc","localhost","fail-9zlsp-small-group","fail-9zlsp-worker-svc.test-ns-k92wt.svc","fail-9zlsp-worker-svc.test-ns-k92wt.svc.cluster.local","*.fail-9zlsp-worker-svc.test-ns-k92wt.svc","*.fail-9zlsp-worker-svc.test-ns-k92wt.svc.cluster.local","*-worker-*.test-ns-k92wt.svc","*-worker-*.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.26","10.132.0.47","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:36:25.409Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"092191ba-f0b9-42c0-9647-86831ac8254c","rayCluster":"fail-9zlsp","podIPs":["10.134.0.26","10.132.0.47"]}
{"level":"info","ts":"2026-06-10T10:36:25.411Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"6b4f561b-f633-40c4-a313-650dedf53d6d","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:25.411Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"6b4f561b-f633-40c4-a313-650dedf53d6d","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:27.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"6168f9eb-8405-4201-ae47-1d5dc7793d72","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:27.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"6168f9eb-8405-4201-ae47-1d5dc7793d72","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:27.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"6168f9eb-8405-4201-ae47-1d5dc7793d72","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:27.301Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"6168f9eb-8405-4201-ae47-1d5dc7793d72"}
{"level":"info","ts":"2026-06-10T10:36:27.301Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"6168f9eb-8405-4201-ae47-1d5dc7793d72","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076"}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076"}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:27.389Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"560cb433-ab47-4645-b53d-9226cb6f1076","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:30.301Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4adbaa2e-9ef6-43e4-94d1-f8cd3211d745","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:30.301Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4adbaa2e-9ef6-43e4-94d1-f8cd3211d745","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:30.301Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4adbaa2e-9ef6-43e4-94d1-f8cd3211d745","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:30.301Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4adbaa2e-9ef6-43e4-94d1-f8cd3211d745"}
{"level":"info","ts":"2026-06-10T10:36:30.301Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4adbaa2e-9ef6-43e4-94d1-f8cd3211d745","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb"}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb"}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","podName":"fail-9zlsp-head-hvld7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:30.394Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"5d931228-4f6f-4752-aed7-e1f0aeddadfb","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c"}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c"}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","podName":"fail-9zlsp-head-hvld7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","podName":"fail-9zlsp-head-hvld7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:36:31.403Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"c5074324-f384-47cc-86e9-cf76864cc49c","name":"fail-9zlsp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:31.414Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"ea1238dd-4a8a-4cd6-9125-a4b1d719c524","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:31.414Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"510508c2-f513-4441-a649-0c01da7175a1","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:31.414Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"ea1238dd-4a8a-4cd6-9125-a4b1d719c524","pod":"fail-9zlsp-head-hvld7","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-10T10:36:31.414Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"ea1238dd-4a8a-4cd6-9125-a4b1d719c524","pod":"fail-9zlsp-small-group-worker-zcgkr","ip":"10.132.0.47"}
{"level":"info","ts":"2026-06-10T10:36:31.414Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"ea1238dd-4a8a-4cd6-9125-a4b1d719c524","rayCluster":"fail-9zlsp","podIPs":["10.134.0.26","10.132.0.47"]}
{"level":"info","ts":"2026-06-10T10:36:31.414Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"5d54a34e-ec30-470f-a2f9-d796f9ac1216","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:31.414Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"5d54a34e-ec30-470f-a2f9-d796f9ac1216","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:31.414Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"5d54a34e-ec30-470f-a2f9-d796f9ac1216","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:31.415Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"5d54a34e-ec30-470f-a2f9-d796f9ac1216"}
{"level":"info","ts":"2026-06-10T10:36:31.415Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"5d54a34e-ec30-470f-a2f9-d796f9ac1216","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:31.426Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"510508c2-f513-4441-a649-0c01da7175a1","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:31.434Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"510508c2-f513-4441-a649-0c01da7175a1","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:31.434Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"510508c2-f513-4441-a649-0c01da7175a1","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:33.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9f1bd392-6d9a-4387-9814-cb730b4eb8eb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:33.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9f1bd392-6d9a-4387-9814-cb730b4eb8eb","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:33.302Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9f1bd392-6d9a-4387-9814-cb730b4eb8eb","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:33.302Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9f1bd392-6d9a-4387-9814-cb730b4eb8eb"}
{"level":"info","ts":"2026-06-10T10:36:33.302Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9f1bd392-6d9a-4387-9814-cb730b4eb8eb","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec"}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec"}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","podName":"fail-9zlsp-head-hvld7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","podName":"fail-9zlsp-head-hvld7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:33.415Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1fa25e35-07ad-4020-b62b-d32d17b32eec","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:33.802Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f"}
{"level":"info","ts":"2026-06-10T10:36:33.802Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f"}
{"level":"info","ts":"2026-06-10T10:36:33.802Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:33.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:33.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:33.802Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:33.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:33.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:33.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:33.803Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:33.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:33.803Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:36:33.803Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:33.803Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"6db8cbba-5206-498a-a47c-b7e9af3b621f","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:34.811Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093"}
{"level":"info","ts":"2026-06-10T10:36:34.811Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093"}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","podName":"fail-9zlsp-head-hvld7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","podName":"fail-9zlsp-head-hvld7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:36:34.812Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"1259281c-a63d-4ba0-9efd-8dce9ef1d093","name":"fail-9zlsp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"5e2eef6b-0ff3-4ca8-8202-0a8b5e62346a","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"97ee122a-9e7e-45dc-a8a2-a3b5105a6dfa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"97ee122a-9e7e-45dc-a8a2-a3b5105a6dfa","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"97ee122a-9e7e-45dc-a8a2-a3b5105a6dfa","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"ac653680-1159-4027-a105-aebb1be53bbf","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"97ee122a-9e7e-45dc-a8a2-a3b5105a6dfa"}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"97ee122a-9e7e-45dc-a8a2-a3b5105a6dfa","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"ac653680-1159-4027-a105-aebb1be53bbf","pod":"fail-9zlsp-head-hvld7","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"ac653680-1159-4027-a105-aebb1be53bbf","pod":"fail-9zlsp-small-group-worker-zcgkr","ip":"10.132.0.47"}
{"level":"info","ts":"2026-06-10T10:36:34.823Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"ac653680-1159-4027-a105-aebb1be53bbf","rayCluster":"fail-9zlsp","podIPs":["10.134.0.26","10.132.0.47"]}
{"level":"info","ts":"2026-06-10T10:36:34.833Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"5e2eef6b-0ff3-4ca8-8202-0a8b5e62346a","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:34.844Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"5e2eef6b-0ff3-4ca8-8202-0a8b5e62346a","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:34.844Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"5e2eef6b-0ff3-4ca8-8202-0a8b5e62346a","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:36.212Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"365debd5-f2d6-4d61-8adc-98e9e9bfed2d","namespace":"test-ns-k92wt","clusterName":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:36:36.212Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-ksbpl","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"counter-ksbpl","reconcileID":"365debd5-f2d6-4d61-8adc-98e9e9bfed2d","namespace":"test-ns-k92wt","clusterName":"counter-ksbpl"}
{"level":"info","ts":"2026-06-10T10:36:36.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a24a00de-0209-4ff5-9d0b-9a2f5e0ebaf0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:36.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a24a00de-0209-4ff5-9d0b-9a2f5e0ebaf0","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:36.302Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a24a00de-0209-4ff5-9d0b-9a2f5e0ebaf0","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:36.303Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a24a00de-0209-4ff5-9d0b-9a2f5e0ebaf0"}
{"level":"info","ts":"2026-06-10T10:36:36.303Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a24a00de-0209-4ff5-9d0b-9a2f5e0ebaf0","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f"}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f"}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","podName":"fail-9zlsp-head-hvld7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","podName":"fail-9zlsp-head-hvld7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:36.824Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"08a43345-673d-4b3d-823c-16a8d139b42f","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:39.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"08fa80c4-935a-460c-a936-1d7bb7af443b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:39.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"08fa80c4-935a-460c-a936-1d7bb7af443b","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:39.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"08fa80c4-935a-460c-a936-1d7bb7af443b","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:39.303Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"08fa80c4-935a-460c-a936-1d7bb7af443b"}
{"level":"info","ts":"2026-06-10T10:36:39.303Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"08fa80c4-935a-460c-a936-1d7bb7af443b","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:42.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f5a2bb88-a36f-461e-9f71-e08651b57ae1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:42.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f5a2bb88-a36f-461e-9f71-e08651b57ae1","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:42.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f5a2bb88-a36f-461e-9f71-e08651b57ae1","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:42.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f5a2bb88-a36f-461e-9f71-e08651b57ae1"}
{"level":"info","ts":"2026-06-10T10:36:42.304Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f5a2bb88-a36f-461e-9f71-e08651b57ae1","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a"}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a"}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:36:42.432Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"572ea92a-4623-4a2d-bbd2-e3a68845c15a","name":"fail-9zlsp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"8dbda7ec-d257-4b07-a158-46a1cf67e4a7","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a3b39314-b57f-477b-b6cb-2a466bc1bc50","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"03dfa534-5148-47d7-82db-219a5ed0a348","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"03dfa534-5148-47d7-82db-219a5ed0a348","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"03dfa534-5148-47d7-82db-219a5ed0a348","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a3b39314-b57f-477b-b6cb-2a466bc1bc50","pod":"fail-9zlsp-head-hvld7","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a3b39314-b57f-477b-b6cb-2a466bc1bc50","pod":"fail-9zlsp-small-group-worker-zcgkr","ip":"10.132.0.47"}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"a3b39314-b57f-477b-b6cb-2a466bc1bc50","rayCluster":"fail-9zlsp","podIPs":["10.134.0.26","10.132.0.47"]}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"03dfa534-5148-47d7-82db-219a5ed0a348"}
{"level":"info","ts":"2026-06-10T10:36:42.446Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"03dfa534-5148-47d7-82db-219a5ed0a348","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:42.455Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"8dbda7ec-d257-4b07-a158-46a1cf67e4a7","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:42.466Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"8dbda7ec-d257-4b07-a158-46a1cf67e4a7","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:42.466Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"8dbda7ec-d257-4b07-a158-46a1cf67e4a7","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48"}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48"}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","podName":"fail-9zlsp-small-group-worker-zcgkr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:36:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:44.446Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"3bd72ef9-fe83-4e6b-8773-0402bc6f3a48","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:45.305Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"0a50a659-a019-442a-9374-bb7c0cf856dd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:45.305Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"0a50a659-a019-442a-9374-bb7c0cf856dd","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.305Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"0a50a659-a019-442a-9374-bb7c0cf856dd","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:45.305Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"0a50a659-a019-442a-9374-bb7c0cf856dd"}
{"level":"info","ts":"2026-06-10T10:36:45.305Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"0a50a659-a019-442a-9374-bb7c0cf856dd","RayCluster":"fail-9zlsp","State":""}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d"}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d"}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:36:45.839Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"8b2bef16-5c87-49a8-83ec-4128595e967d","name":"fail-9zlsp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:36:45.851Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"6dabd427-a9aa-4e3f-a2b5-174cc3f59cee","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.851Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"2846907c-164a-4a59-a464-d76a6797c465","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"6dabd427-a9aa-4e3f-a2b5-174cc3f59cee","pod":"fail-9zlsp-small-group-worker-zcgkr","ip":"10.132.0.47"}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"6dabd427-a9aa-4e3f-a2b5-174cc3f59cee","pod":"fail-9zlsp-head-hvld7","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"6dabd427-a9aa-4e3f-a2b5-174cc3f59cee","rayCluster":"fail-9zlsp","podIPs":["10.132.0.47","10.134.0.26"]}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5"}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","head service name":"fail-9zlsp-head-svc","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","head service URL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5"}
{"level":"info","ts":"2026-06-10T10:36:45.852Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","command":["if","!","ray","job","status","--address","http://fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","fail-qtsqd",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","--no-wait","--submission-id","fail-qtsqd","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","--follow","fail-qtsqd"]}
{"level":"info","ts":"2026-06-10T10:36:45.862Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-06-10T10:36:45.862Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","SubmissionMode":"K8sJobMode","RayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.862Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:36:45.862Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e91adc05-cdb0-49c9-8c46-1bae6d5259e5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:36:45.867Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"2846907c-164a-4a59-a464-d76a6797c465","name":"fail-9zlsp-head"}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0"}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","head service name":"fail-9zlsp-head-svc","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","head service URL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","SubmissionMode":"K8sJobMode","RayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:36:45.873Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:36:45.899Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"2846907c-164a-4a59-a464-d76a6797c465","name":"fail-9zlsp-workers"}
{"level":"info","ts":"2026-06-10T10:36:45.899Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"2846907c-164a-4a59-a464-d76a6797c465","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.899Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-10T10:36:45.899Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0"}
{"level":"error","ts":"2026-06-10T10:36:45.899Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3ee0198c-f68a-410e-ae48-cfd7c41c99c0","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:45.899Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2cdd8f3a-23ea-4150-9527-2bfa9fb92c6f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:45.899Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2cdd8f3a-23ea-4150-9527-2bfa9fb92c6f","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.899Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2cdd8f3a-23ea-4150-9527-2bfa9fb92c6f","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:45.899Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2cdd8f3a-23ea-4150-9527-2bfa9fb92c6f"}
{"level":"error","ts":"2026-06-10T10:36:45.911Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2cdd8f3a-23ea-4150-9527-2bfa9fb92c6f","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:45.911Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2cdd8f3a-23ea-4150-9527-2bfa9fb92c6f"}
{"level":"error","ts":"2026-06-10T10:36:45.911Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2cdd8f3a-23ea-4150-9527-2bfa9fb92c6f","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:45.911Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9ce026f4-8936-4815-889c-57223d588fb5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:45.911Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9ce026f4-8936-4815-889c-57223d588fb5","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.911Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9ce026f4-8936-4815-889c-57223d588fb5","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:45.911Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9ce026f4-8936-4815-889c-57223d588fb5"}
{"level":"error","ts":"2026-06-10T10:36:45.916Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9ce026f4-8936-4815-889c-57223d588fb5","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:45.916Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9ce026f4-8936-4815-889c-57223d588fb5"}
{"level":"error","ts":"2026-06-10T10:36:45.916Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9ce026f4-8936-4815-889c-57223d588fb5","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:45.921Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1fda5648-124e-4865-98c0-a21543005525","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:45.921Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1fda5648-124e-4865-98c0-a21543005525","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.921Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1fda5648-124e-4865-98c0-a21543005525","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:45.921Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1fda5648-124e-4865-98c0-a21543005525"}
{"level":"error","ts":"2026-06-10T10:36:45.926Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1fda5648-124e-4865-98c0-a21543005525","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:45.926Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1fda5648-124e-4865-98c0-a21543005525"}
{"level":"error","ts":"2026-06-10T10:36:45.926Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1fda5648-124e-4865-98c0-a21543005525","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:45.966Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3aa29768-f573-45d1-8b76-3130c9750323","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:45.966Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3aa29768-f573-45d1-8b76-3130c9750323","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:45.966Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3aa29768-f573-45d1-8b76-3130c9750323","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:45.966Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3aa29768-f573-45d1-8b76-3130c9750323"}
{"level":"error","ts":"2026-06-10T10:36:45.972Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3aa29768-f573-45d1-8b76-3130c9750323","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:45.972Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3aa29768-f573-45d1-8b76-3130c9750323"}
{"level":"error","ts":"2026-06-10T10:36:45.972Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"3aa29768-f573-45d1-8b76-3130c9750323","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:46.052Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c6ef5eb-c9db-40dd-9116-4409d54d20eb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:46.052Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c6ef5eb-c9db-40dd-9116-4409d54d20eb","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:46.052Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c6ef5eb-c9db-40dd-9116-4409d54d20eb","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:46.052Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c6ef5eb-c9db-40dd-9116-4409d54d20eb"}
{"level":"error","ts":"2026-06-10T10:36:46.058Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c6ef5eb-c9db-40dd-9116-4409d54d20eb","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:46.058Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c6ef5eb-c9db-40dd-9116-4409d54d20eb"}
{"level":"error","ts":"2026-06-10T10:36:46.058Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c6ef5eb-c9db-40dd-9116-4409d54d20eb","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:46.219Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1135c482-fdc0-4b97-86a6-d23b6c3fdfa6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:46.219Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1135c482-fdc0-4b97-86a6-d23b6c3fdfa6","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:46.219Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1135c482-fdc0-4b97-86a6-d23b6c3fdfa6","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:46.219Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1135c482-fdc0-4b97-86a6-d23b6c3fdfa6"}
{"level":"error","ts":"2026-06-10T10:36:46.225Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1135c482-fdc0-4b97-86a6-d23b6c3fdfa6","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:46.225Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1135c482-fdc0-4b97-86a6-d23b6c3fdfa6"}
{"level":"error","ts":"2026-06-10T10:36:46.225Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"1135c482-fdc0-4b97-86a6-d23b6c3fdfa6","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:46.546Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"aaca9abb-a42a-4636-bd07-fb3129d76bd7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:46.546Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"aaca9abb-a42a-4636-bd07-fb3129d76bd7","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:46.546Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"aaca9abb-a42a-4636-bd07-fb3129d76bd7","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:46.546Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"aaca9abb-a42a-4636-bd07-fb3129d76bd7"}
{"level":"error","ts":"2026-06-10T10:36:46.551Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"aaca9abb-a42a-4636-bd07-fb3129d76bd7","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:46.551Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"aaca9abb-a42a-4636-bd07-fb3129d76bd7"}
{"level":"error","ts":"2026-06-10T10:36:46.551Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"aaca9abb-a42a-4636-bd07-fb3129d76bd7","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:46.888Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"7c51805c-bdb1-4023-a1d3-0c1d2de4748f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:46.888Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"7c51805c-bdb1-4023-a1d3-0c1d2de4748f","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:46.888Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"7c51805c-bdb1-4023-a1d3-0c1d2de4748f","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:46.888Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"7c51805c-bdb1-4023-a1d3-0c1d2de4748f"}
{"level":"error","ts":"2026-06-10T10:36:46.893Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"7c51805c-bdb1-4023-a1d3-0c1d2de4748f","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:46.893Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"7c51805c-bdb1-4023-a1d3-0c1d2de4748f"}
{"level":"error","ts":"2026-06-10T10:36:46.893Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"7c51805c-bdb1-4023-a1d3-0c1d2de4748f","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:46.914Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7284b175-f419-4903-b90d-562b8a474b4b","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:36:46.914Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7284b175-f419-4903-b90d-562b8a474b4b","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:36:46.914Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7284b175-f419-4903-b90d-562b8a474b4b","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:36:46.914Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7284b175-f419-4903-b90d-562b8a474b4b","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:36:47.192Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c5621e8-9c12-4b89-b447-930dd913ea8d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:47.192Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c5621e8-9c12-4b89-b447-930dd913ea8d","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:47.192Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c5621e8-9c12-4b89-b447-930dd913ea8d","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:47.193Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c5621e8-9c12-4b89-b447-930dd913ea8d"}
{"level":"error","ts":"2026-06-10T10:36:47.198Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c5621e8-9c12-4b89-b447-930dd913ea8d","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:47.198Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c5621e8-9c12-4b89-b447-930dd913ea8d"}
{"level":"error","ts":"2026-06-10T10:36:47.198Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"4c5621e8-9c12-4b89-b447-930dd913ea8d","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:47.852Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934"}
{"level":"info","ts":"2026-06-10T10:36:47.852Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934"}
{"level":"info","ts":"2026-06-10T10:36:47.852Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","1 head service found":"fail-9zlsp-head-svc"}
{"level":"info","ts":"2026-06-10T10:36:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","Found 1 head Pod":"fail-9zlsp-head-hvld7","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:36:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","head Pod":"fail-9zlsp-head-hvld7","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-9zlsp-head-hvld7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:47.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:36:47.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","worker Pod":"fail-9zlsp-small-group-worker-zcgkr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-9zlsp-small-group-worker-zcgkr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:36:47.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:47.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:36:47.853Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:36:47.853Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:36:47.853Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"06f40e40-0c92-433f-9b10-150fb8cc3934","seconds":300}
{"level":"info","ts":"2026-06-10T10:36:49.759Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f7d3317f-e24d-4a3b-849d-cc8f5d7ed768","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:49.759Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f7d3317f-e24d-4a3b-849d-cc8f5d7ed768","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:49.759Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f7d3317f-e24d-4a3b-849d-cc8f5d7ed768","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:49.759Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f7d3317f-e24d-4a3b-849d-cc8f5d7ed768"}
{"level":"error","ts":"2026-06-10T10:36:49.765Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f7d3317f-e24d-4a3b-849d-cc8f5d7ed768","JobId":"fail-qtsqd","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:49.765Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f7d3317f-e24d-4a3b-849d-cc8f5d7ed768"}
{"level":"error","ts":"2026-06-10T10:36:49.765Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"f7d3317f-e24d-4a3b-849d-cc8f5d7ed768","error":"Job fail-qtsqd does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:36:51.522Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"d7a196a5-1c9c-42c5-bbe3-a2cb76019858","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:51.522Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"d7a196a5-1c9c-42c5-bbe3-a2cb76019858","pod":"fail-9zlsp-head-hvld7","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-10T10:36:51.522Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"d7a196a5-1c9c-42c5-bbe3-a2cb76019858","pod":"fail-9zlsp-small-group-worker-zcgkr","ip":"10.132.0.47"}
{"level":"info","ts":"2026-06-10T10:36:51.522Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"d7a196a5-1c9c-42c5-bbe3-a2cb76019858","rayCluster":"fail-9zlsp","podIPs":["10.134.0.26","10.132.0.47"]}
{"level":"info","ts":"2026-06-10T10:36:54.885Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a96af644-d083-4d42-bf92-763b4f489850","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:54.885Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a96af644-d083-4d42-bf92-763b4f489850","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:54.885Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a96af644-d083-4d42-bf92-763b4f489850","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:54.885Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a96af644-d083-4d42-bf92-763b4f489850"}
{"level":"info","ts":"2026-06-10T10:36:54.892Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a96af644-d083-4d42-bf92-763b4f489850","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:36:54.892Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"a96af644-d083-4d42-bf92-763b4f489850","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:36:54.913Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e1875fb1-d3a4-4423-8963-374fdfd2cde3","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:54.913Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e1875fb1-d3a4-4423-8963-374fdfd2cde3","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:54.913Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e1875fb1-d3a4-4423-8963-374fdfd2cde3","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:54.913Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e1875fb1-d3a4-4423-8963-374fdfd2cde3"}
{"level":"info","ts":"2026-06-10T10:36:54.916Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"e1875fb1-d3a4-4423-8963-374fdfd2cde3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:36:57.913Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bfd3ae7b-b62a-4ad0-86c6-049c18487826","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:57.913Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bfd3ae7b-b62a-4ad0-86c6-049c18487826","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:57.913Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bfd3ae7b-b62a-4ad0-86c6-049c18487826","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:57.913Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bfd3ae7b-b62a-4ad0-86c6-049c18487826"}
{"level":"info","ts":"2026-06-10T10:36:57.917Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bfd3ae7b-b62a-4ad0-86c6-049c18487826","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:36:57.917Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"bfd3ae7b-b62a-4ad0-86c6-049c18487826","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:36:57.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"760a5a47-782c-45de-9014-5061cbcd43ad","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:36:57.930Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"760a5a47-782c-45de-9014-5061cbcd43ad","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:36:57.930Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"760a5a47-782c-45de-9014-5061cbcd43ad","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:36:57.931Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"760a5a47-782c-45de-9014-5061cbcd43ad"}
{"level":"info","ts":"2026-06-10T10:36:57.933Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"760a5a47-782c-45de-9014-5061cbcd43ad","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:37:00.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"ae558dce-f74b-49f0-99fd-e9a50fa3f620","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:00.930Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"ae558dce-f74b-49f0-99fd-e9a50fa3f620","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:00.930Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"ae558dce-f74b-49f0-99fd-e9a50fa3f620","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:00.930Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"ae558dce-f74b-49f0-99fd-e9a50fa3f620"}
{"level":"info","ts":"2026-06-10T10:37:00.933Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"ae558dce-f74b-49f0-99fd-e9a50fa3f620","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:37:03.934Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"cb7a5446-8213-4e7f-82ac-8c617c4e62f2","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:03.934Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"cb7a5446-8213-4e7f-82ac-8c617c4e62f2","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:03.934Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"cb7a5446-8213-4e7f-82ac-8c617c4e62f2","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:03.934Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"cb7a5446-8213-4e7f-82ac-8c617c4e62f2"}
{"level":"info","ts":"2026-06-10T10:37:03.937Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"cb7a5446-8213-4e7f-82ac-8c617c4e62f2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:37:04.523Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"822ef98c-af36-4cc6-bada-7200602e1631","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:04.523Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"822ef98c-af36-4cc6-bada-7200602e1631","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:04.523Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"822ef98c-af36-4cc6-bada-7200602e1631","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:04.523Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"822ef98c-af36-4cc6-bada-7200602e1631"}
{"level":"info","ts":"2026-06-10T10:37:04.526Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"822ef98c-af36-4cc6-bada-7200602e1631","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:37:05.530Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"81549c4b-a9e8-4b94-acbd-cfaf332de158","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:05.530Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"81549c4b-a9e8-4b94-acbd-cfaf332de158","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:05.530Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"81549c4b-a9e8-4b94-acbd-cfaf332de158","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:05.530Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"81549c4b-a9e8-4b94-acbd-cfaf332de158"}
{"level":"info","ts":"2026-06-10T10:37:05.533Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"81549c4b-a9e8-4b94-acbd-cfaf332de158","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:37:05.547Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"586ba034-95df-45d9-943f-99a65b0fb2b9","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:05.547Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"586ba034-95df-45d9-943f-99a65b0fb2b9","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:05.547Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"586ba034-95df-45d9-943f-99a65b0fb2b9","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:05.547Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"586ba034-95df-45d9-943f-99a65b0fb2b9"}
{"level":"info","ts":"2026-06-10T10:37:05.550Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"586ba034-95df-45d9-943f-99a65b0fb2b9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:36:53Z","endTime":"2026-06-10T10:36:55Z"},"jobId":"fail-qtsqd","rayClusterName":"fail-9zlsp","dashboardURL":"fail-9zlsp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:36:53,444\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-qtsqd: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:36:21Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:36:45Z","stateTransitionTimes":{"ready":"2026-06-10T10:36:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.26","serviceIP":"10.134.0.26","podName":"fail-9zlsp-head-hvld7","serviceName":"fail-9zlsp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:36:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:36:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:36:21Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:36:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:37:05.550Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"586ba034-95df-45d9-943f-99a65b0fb2b9","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-10T10:37:05.563Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"ecc64a71-0d7f-4534-b0de-74cf7a501c75","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:05.563Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"ecc64a71-0d7f-4534-b0de-74cf7a501c75","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:37:05 +0000 UTC","Now":"2026-06-10T10:37:05.563Z","ShutdownTime":"2026-06-10T10:37:05.000Z"}
{"level":"info","ts":"2026-06-10T10:37:06.525Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9647928a-ee5a-4ddf-82f6-a6a48d6ef7a3","DeletionTimestamp":"2026-06-10 10:37:06 +0000 UTC"}
{"level":"info","ts":"2026-06-10T10:37:06.525Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"9647928a-ee5a-4ddf-82f6-a6a48d6ef7a3","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:37:06.539Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"ac276a19-e260-4b46-ba56-b9d04c69fcd7"}
{"level":"info","ts":"2026-06-10T10:37:06.551Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"2607723c-73b3-4c76-b06e-ccc197595a87"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1a7efdc0-fc68-48cb-bd6d-d16e55ed937f","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1a7efdc0-fc68-48cb-bd6d-d16e55ed937f","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1a7efdc0-fc68-48cb-bd6d-d16e55ed937f","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"reconcileID":"e63c00a1-8fdc-4db8-b1c7-e54f935ff0c6"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"3c43bac9-2533-4f2e-8644-7b47eebbf4d5"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1a7efdc0-fc68-48cb-bd6d-d16e55ed937f","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"521a5424-e6b8-42c0-beb3-105ac1c4534a","rayCluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"19e54261-dbe1-4e29-8e88-30ff6641032e"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"521a5424-e6b8-42c0-beb3-105ac1c4534a","pod":"fail-9zlsp-head-hvld7","ip":"10.134.0.26"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"521a5424-e6b8-42c0-beb3-105ac1c4534a","pod":"fail-9zlsp-small-group-worker-zcgkr","ip":"10.132.0.47"}
{"level":"info","ts":"2026-06-10T10:37:06.557Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"521a5424-e6b8-42c0-beb3-105ac1c4534a","rayCluster":"fail-9zlsp","podIPs":["10.134.0.26","10.132.0.47"]}
{"level":"info","ts":"2026-06-10T10:37:06.564Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1a7efdc0-fc68-48cb-bd6d-d16e55ed937f","name":"kuberay-gateway-access","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:37:06.572Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1a7efdc0-fc68-48cb-bd6d-d16e55ed937f","name":"test-ns-k92wt-fail-9zlsp","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:37:06.577Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1a7efdc0-fc68-48cb-bd6d-d16e55ed937f","configMap":"kube-rbac-proxy-config-fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.581Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1a7efdc0-fc68-48cb-bd6d-d16e55ed937f","serviceAccount":"fail-9zlsp-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:37:06.589Z","logger":"raycluster-resource","msg":"default","name":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.589Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-9zlsp","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:37:06.599Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"1a7efdc0-fc68-48cb-bd6d-d16e55ed937f","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.599Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cd8c6130-8354-42fb-b5e4-f714c1808216","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:06.599Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cd8c6130-8354-42fb-b5e4-f714c1808216","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.599Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cd8c6130-8354-42fb-b5e4-f714c1808216","cluster":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.599Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cd8c6130-8354-42fb-b5e4-f714c1808216","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:37:06.599Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cd8c6130-8354-42fb-b5e4-f714c1808216","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:37:06.599Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"89a4b8c0-ee58-4112-9ea2-b72cd3a91012","namespace":"test-ns-k92wt","clusterName":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.599Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"6bf07c7d-07b9-4a05-b190-a2d4d00647b9"}
{"level":"info","ts":"2026-06-10T10:37:06.599Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"589f4fed-6125-4da1-bfad-c031836c0d18"}
{"level":"error","ts":"2026-06-10T10:37:06.605Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cd8c6130-8354-42fb-b5e4-f714c1808216","error":"rayclusters.ray.io \"fail-9zlsp\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:06.605Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cd8c6130-8354-42fb-b5e4-f714c1808216"}
{"level":"error","ts":"2026-06-10T10:37:06.605Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cd8c6130-8354-42fb-b5e4-f714c1808216","error":"rayclusters.ray.io \"fail-9zlsp\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:06.605Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cb02c659-6a7f-47ca-9b4f-58256e2b8ac4","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:06.605Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cb02c659-6a7f-47ca-9b4f-58256e2b8ac4"}
{"level":"info","ts":"2026-06-10T10:37:06.605Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cb02c659-6a7f-47ca-9b4f-58256e2b8ac4","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:37:06.605Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"cb02c659-6a7f-47ca-9b4f-58256e2b8ac4","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:37:06.607Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"9ce5d026-f86f-47da-aa37-05a77168e27f"}
{"level":"info","ts":"2026-06-10T10:37:06.610Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"255c289f-7402-444f-a4a5-cc169ba25f28","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:06.610Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"255c289f-7402-444f-a4a5-cc169ba25f28"}
{"level":"info","ts":"2026-06-10T10:37:06.610Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"255c289f-7402-444f-a4a5-cc169ba25f28","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:37:06.610Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"255c289f-7402-444f-a4a5-cc169ba25f28","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:37:06.610Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"78dc3d45-5ac3-4859-85dc-5100adb13cd6"}
{"level":"info","ts":"2026-06-10T10:37:06.629Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"4168febf-c679-4420-b8f1-8414bb6266c6","namespacedName":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:06.629Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"4168febf-c679-4420-b8f1-8414bb6266c6"}
{"level":"info","ts":"2026-06-10T10:37:06.629Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"4168febf-c679-4420-b8f1-8414bb6266c6","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:37:06.629Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"4168febf-c679-4420-b8f1-8414bb6266c6","namespace":"test-ns-k92wt"}
{"level":"error","ts":"2026-06-10T10:37:06.632Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"89a4b8c0-ee58-4112-9ea2-b72cd3a91012","certificate":"ray-worker-cert-fail-9zlsp","error":"certificates.cert-manager.io \"ray-worker-cert-fail-9zlsp\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:06.699Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"89a4b8c0-ee58-4112-9ea2-b72cd3a91012","errorCount":1}
{"level":"info","ts":"2026-06-10T10:37:06.699Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"89a4b8c0-ee58-4112-9ea2-b72cd3a91012"}
{"level":"error","ts":"2026-06-10T10:37:06.699Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"89a4b8c0-ee58-4112-9ea2-b72cd3a91012","error":"certificates.cert-manager.io \"ray-worker-cert-fail-9zlsp\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:06.704Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"55f1e2d3-876f-415a-825d-f978208049f4","namespace":"test-ns-k92wt","clusterName":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.704Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-9zlsp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-9zlsp","reconcileID":"55f1e2d3-876f-415a-825d-f978208049f4","namespace":"test-ns-k92wt","clusterName":"fail-9zlsp"}
{"level":"info","ts":"2026-06-10T10:37:06.938Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail","namespace":"test-ns-k92wt"},"reconcileID":"28d332b9-cbb5-45ab-8eb6-736c168bac68"}
{"level":"info","ts":"2026-06-10T10:37:07.557Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c8402134-5809-443a-b181-7f8fbb64e8d0","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:07.557Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c8402134-5809-443a-b181-7f8fbb64e8d0","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:37:07.571Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c8402134-5809-443a-b181-7f8fbb64e8d0"}
{"level":"info","ts":"2026-06-10T10:37:07.571Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c8402134-5809-443a-b181-7f8fbb64e8d0","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:37:07.571Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c8402134-5809-443a-b181-7f8fbb64e8d0","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-w6wqk","rayClusterName":"fail-k8s-job-txc8t","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:37:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:37:07.571Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c8402134-5809-443a-b181-7f8fbb64e8d0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:37:07.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"0984d005-872b-42fe-84ce-1892526f78b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:07.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"0984d005-872b-42fe-84ce-1892526f78b4","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.585Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"0984d005-872b-42fe-84ce-1892526f78b4","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.585Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"0984d005-872b-42fe-84ce-1892526f78b4","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.589Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.590Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:37:07.604Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.604Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","namespacedName":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.604Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"b83357a3-2a58-4d4f-b0fd-c6acb807b18a","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","mode":"IntegratedOAuth","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.604Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809"}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809"}
{"level":"info","ts":"2026-06-10T10:37:07.604Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"0984d005-872b-42fe-84ce-1892526f78b4","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"0984d005-872b-42fe-84ce-1892526f78b4"}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"0984d005-872b-42fe-84ce-1892526f78b4","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed6a57c4-fc0e-4903-be36-fe43c52f8d0c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed6a57c4-fc0e-4903-be36-fe43c52f8d0c","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed6a57c4-fc0e-4903-be36-fe43c52f8d0c","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed6a57c4-fc0e-4903-be36-fe43c52f8d0c"}
{"level":"info","ts":"2026-06-10T10:37:07.605Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed6a57c4-fc0e-4903-be36-fe43c52f8d0c","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:07.610Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.610Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:37:07.613Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","name":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:07.613Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","error":"head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"error","ts":"2026-06-10T10:37:07.613Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.613Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:07.613Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.613Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.613Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.613Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:37:07.613Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:07.614Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.614Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.616Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"b83357a3-2a58-4d4f-b0fd-c6acb807b18a","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.616Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d762df89-22c6-4add-a41e-ea8667eb0977","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.621Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b6b7d793-dd0e-4c3f-95c5-9bdc413882b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:07.621Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b6b7d793-dd0e-4c3f-95c5-9bdc413882b2","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.621Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b6b7d793-dd0e-4c3f-95c5-9bdc413882b2","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.622Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b6b7d793-dd0e-4c3f-95c5-9bdc413882b2"}
{"level":"info","ts":"2026-06-10T10:37:07.622Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b6b7d793-dd0e-4c3f-95c5-9bdc413882b2","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:07.622Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304"}
{"level":"info","ts":"2026-06-10T10:37:07.622Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304","rayCluster":"fail-k8s-job-txc8t","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:37:07.622Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.627Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","name":"fail-k8s-job-txc8t","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-txc8t\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t"}}
{"level":"info","ts":"2026-06-10T10:37:07.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809"}
{"level":"error","ts":"2026-06-10T10:37:07.627Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"12dc754b-605e-47b3-af09-f091cffee809","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3"}
{"level":"info","ts":"2026-06-10T10:37:07.627Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3"}
{"level":"info","ts":"2026-06-10T10:37:07.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:07.627Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","error":"head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"error","ts":"2026-06-10T10:37:07.627Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.627Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:07.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.628Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.628Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:37:07.628Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:07.628Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","name":"fail-k8s-job-txc8t-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:37:07.630Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.699Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d762df89-22c6-4add-a41e-ea8667eb0977","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:07.699Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","name":"kuberay-gateway-access","namespace":"test-ns-k92wt","operation":"created"}
{"level":"info","ts":"2026-06-10T10:37:07.699Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3"}
{"level":"error","ts":"2026-06-10T10:37:07.699Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"e74f44db-7ae3-4c8f-aa8a-483def8c84d3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20","error":"head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"error","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"08d7335e-97e6-49a2-ac52-b968c3ab270c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"08d7335e-97e6-49a2-ac52-b968c3ab270c","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"08d7335e-97e6-49a2-ac52-b968c3ab270c","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20"}
{"level":"error","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"93c8834c-450d-4bdd-93f2-760d67981d20","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"08d7335e-97e6-49a2-ac52-b968c3ab270c"}
{"level":"info","ts":"2026-06-10T10:37:07.700Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"08d7335e-97e6-49a2-ac52-b968c3ab270c","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:07.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3"}
{"level":"info","ts":"2026-06-10T10:37:07.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3"}
{"level":"info","ts":"2026-06-10T10:37:07.799Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304"}
{"level":"info","ts":"2026-06-10T10:37:07.800Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304","rayCluster":"fail-k8s-job-txc8t","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:37:07.800Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:07.800Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3","error":"head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:07.800Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","name":"test-ns-k92wt-fail-k8s-job-txc8t","operation":"created"}
{"level":"error","ts":"2026-06-10T10:37:07.800Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.800Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:07.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.800Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.800Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3"}
{"level":"error","ts":"2026-06-10T10:37:07.800Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"d1132c34-2803-4faf-b5b6-88d7c4987fa3","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.806Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d762df89-22c6-4add-a41e-ea8667eb0977","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:07.806Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d762df89-22c6-4add-a41e-ea8667eb0977","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.806Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"48a043a1-47ec-4dec-a7ec-dd7fdb7d3859","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.815Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","name":"kube-rbac-proxy-config-fail-k8s-job-txc8t","operation":"created"}
{"level":"error","ts":"2026-06-10T10:37:07.824Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-txc8t\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.824Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c"}
{"level":"error","ts":"2026-06-10T10:37:07.824Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"242a3ea7-ae0e-419b-b724-ba3235abbb4c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-txc8t\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.824Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"f6a4e7a5-b8aa-4d0d-9296-bde34ec0dd7c","namespacedName":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.824Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"f6a4e7a5-b8aa-4d0d-9296-bde34ec0dd7c","mode":"IntegratedOAuth","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.824Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"f6a4e7a5-b8aa-4d0d-9296-bde34ec0dd7c","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.824Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304"}
{"level":"info","ts":"2026-06-10T10:37:07.824Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304","rayCluster":"fail-k8s-job-txc8t","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:37:07.824Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d9a45144-23db-4134-bda3-0f5f86119304"}
{"level":"info","ts":"2026-06-10T10:37:07.824Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"edebbb6e-caa1-421a-bd27-bf7e747a454d","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.824Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"edebbb6e-caa1-421a-bd27-bf7e747a454d"}
{"level":"info","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b"}
{"level":"info","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b"}
{"level":"info","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b","error":"head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"error","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b"}
{"level":"error","ts":"2026-06-10T10:37:07.841Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"90ccf274-a0d2-48b1-a64f-326a9f49734b","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.903Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"48a043a1-47ec-4dec-a7ec-dd7fdb7d3859","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"f6a4e7a5-b8aa-4d0d-9296-bde34ec0dd7c","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2e8fbe4f-9746-4791-92ef-67119bbec610","namespacedName":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2e8fbe4f-9746-4791-92ef-67119bbec610","mode":"IntegratedOAuth","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2e8fbe4f-9746-4791-92ef-67119bbec610","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2b82c8c2-6c4a-47e7-a25f-d27f5b85b1b0","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"622175a7-b70f-4998-b62c-a490d6fccba4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"622175a7-b70f-4998-b62c-a490d6fccba4","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2b82c8c2-6c4a-47e7-a25f-d27f5b85b1b0"}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"622175a7-b70f-4998-b62c-a490d6fccba4","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"622175a7-b70f-4998-b62c-a490d6fccba4"}
{"level":"info","ts":"2026-06-10T10:37:07.911Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"622175a7-b70f-4998-b62c-a490d6fccba4","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:07.916Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"48a043a1-47ec-4dec-a7ec-dd7fdb7d3859","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:07.916Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"48a043a1-47ec-4dec-a7ec-dd7fdb7d3859","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.916Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ca363d8e-83d8-479f-943c-e86cbec0fa49","cluster":"fail-k8s-job-txc8t"}
{"level":"error","ts":"2026-06-10T10:37:07.918Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2e8fbe4f-9746-4791-92ef-67119bbec610","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-txc8t\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.919Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2e8fbe4f-9746-4791-92ef-67119bbec610"}
{"level":"error","ts":"2026-06-10T10:37:07.919Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2e8fbe4f-9746-4791-92ef-67119bbec610","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-txc8t\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5"}
{"level":"info","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5"}
{"level":"info","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5","error":"head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"error","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5"}
{"level":"error","ts":"2026-06-10T10:37:07.922Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f5e43943-d52c-4882-a7e2-57757266b0c5","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:07.924Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d6694f74-0807-4965-995f-128813f07246","namespacedName":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:07.924Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d6694f74-0807-4965-995f-128813f07246","mode":"IntegratedOAuth","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.924Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d6694f74-0807-4965-995f-128813f07246","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:07.928Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ca363d8e-83d8-479f-943c-e86cbec0fa49","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:07.932Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d6694f74-0807-4965-995f-128813f07246","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:08.004Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ca363d8e-83d8-479f-943c-e86cbec0fa49","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:08.004Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ca363d8e-83d8-479f-943c-e86cbec0fa49","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177"}
{"level":"info","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177"}
{"level":"info","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177","error":"head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"error","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177"}
{"level":"error","ts":"2026-06-10T10:37:08.083Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"2c1bdbd8-ea42-4c7d-a7d4-dfd9e254d177","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7"}
{"level":"info","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7"}
{"level":"info","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7","error":"head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"error","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7"}
{"level":"error","ts":"2026-06-10T10:37:08.404Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"7630d31d-6cc0-48aa-8a4f-959f70675aa7","error":"mTLS secrets not ready: head certificate secret ray-head-secret-fail-k8s-job-txc8t not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c"}
{"level":"info","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c"}
{"level":"info","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c","error":"worker certificate secret ray-worker-secret-fail-k8s-job-txc8t not found"}
{"level":"error","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-fail-k8s-job-txc8t not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-fail-k8s-job-txc8t not found"}
{"level":"info","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c","filter labels":["test-ns-k92wt",{"ray.io/cluster":"fail-k8s-job-txc8t","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c"}
{"level":"error","ts":"2026-06-10T10:37:09.045Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"c244e770-8e9d-4fd7-8c91-dad14d38fd2c","error":"mTLS secrets not ready: worker certificate secret ray-worker-secret-fail-k8s-job-txc8t not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:10.325Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25"}
{"level":"info","ts":"2026-06-10T10:37:10.325Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25"}
{"level":"info","ts":"2026-06-10T10:37:10.325Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25"}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25"}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","mode":"IntegratedOAuth","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","cluster":"fail-k8s-job-txc8t","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","cluster":"fail-k8s-job-txc8t","authType":"IntegratedOAuth","serviceAccount":"fail-k8s-job-txc8t-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-k8s-job-txc8t","ray.io/group":"headgroup","ray.io/identifier":"fail-k8s-job-txc8t-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:37:10.326Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","enabled":true}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","name":"fail-k8s-job-txc8t-head-j9ptk"}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25"}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","nodeType":"worker","rayStartParams":{"address":"fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","rayStartCmd":"ray start  --address=fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:37:10.343Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","enabled":true}
{"level":"info","ts":"2026-06-10T10:37:10.357Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","name":"fail-k8s-job-txc8t-small-group-worker-t78sr"}
{"level":"info","ts":"2026-06-10T10:37:10.357Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.357Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.357Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-k8s-job-txc8t-head-svc"},"newHead":{"podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"}}
{"level":"info","ts":"2026-06-10T10:37:10.357Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"577176d5-5db8-462a-bfdb-a25f57735e25","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"9ec0266f-ec13-4046-9da4-691f449130a1","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ce279d3b-1483-4388-8d18-8994b2a581d6","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e46fd947-6dd3-4759-82a3-c3ab90782d8d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e46fd947-6dd3-4759-82a3-c3ab90782d8d","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e46fd947-6dd3-4759-82a3-c3ab90782d8d","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ce279d3b-1483-4388-8d18-8994b2a581d6","rayCluster":"fail-k8s-job-txc8t","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"fail-k8s-job-txc8t-head-svc"},"newHead":{"podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"}}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e46fd947-6dd3-4759-82a3-c3ab90782d8d"}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e46fd947-6dd3-4759-82a3-c3ab90782d8d","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:10.371Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:10.380Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"9ec0266f-ec13-4046-9da4-691f449130a1","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","name":"fail-k8s-job-txc8t","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-txc8t\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t"}}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083"}
{"level":"error","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ba61e67a-cf92-4375-9425-8b0a7b386083","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-txc8t\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:37:10.399Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1f22a955-c499-459b-9b2a-06d89f0fc741","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:10.406Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"9ec0266f-ec13-4046-9da4-691f449130a1","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:10.406Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"9ec0266f-ec13-4046-9da4-691f449130a1","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7a6dad30-e52a-47a6-bf7b-4c38e6a9b5e0","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d19615d8-0cbe-4688-9416-588eb7a585c6","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"59033765-307b-4e9c-8f5b-f3253215a145","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7a6dad30-e52a-47a6-bf7b-4c38e6a9b5e0","rayCluster":"fail-k8s-job-txc8t","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"59033765-307b-4e9c-8f5b-f3253215a145","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"59033765-307b-4e9c-8f5b-f3253215a145","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"59033765-307b-4e9c-8f5b-f3253215a145"}
{"level":"info","ts":"2026-06-10T10:37:10.410Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"59033765-307b-4e9c-8f5b-f3253215a145","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:10.411Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.411Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:10.411Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6c6aa1d6-e20d-4bc6-90a1-343f84689cb9","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:10.419Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d19615d8-0cbe-4688-9416-588eb7a585c6","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:10.430Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d19615d8-0cbe-4688-9416-588eb7a585c6","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:10.430Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d19615d8-0cbe-4688-9416-588eb7a585c6","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"18f2a7d3-48e0-4a02-af47-2ab708470344","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:10.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"18f2a7d3-48e0-4a02-af47-2ab708470344","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:10.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"18f2a7d3-48e0-4a02-af47-2ab708470344","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:10.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"18f2a7d3-48e0-4a02-af47-2ab708470344"}
{"level":"info","ts":"2026-06-10T10:37:10.585Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"18f2a7d3-48e0-4a02-af47-2ab708470344","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9"}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9"}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:10.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.795Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.795Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.795Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:10.795Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"21a445cb-18fa-499d-8710-bf5b5d6ef7a9","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:10.830Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e"}
{"level":"info","ts":"2026-06-10T10:37:10.830Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e"}
{"level":"info","ts":"2026-06-10T10:37:10.830Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:10.831Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"84e767d4-0b60-46cb-b596-9ca4e61fa78e","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca"}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca"}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:11.001Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"9de4e69e-b00f-419d-93f1-ab57cc16f1ca","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:11.558Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910"}
{"level":"info","ts":"2026-06-10T10:37:11.558Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910"}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"newHead":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"}}
{"level":"info","ts":"2026-06-10T10:37:11.559Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"07290673-1c38-4163-914f-d3be33fbe910","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:11.572Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"bab163f6-48f7-4002-a25d-f83fd03d2213","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:11.572Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"dafda1d9-aa98-497a-9004-7bf0ac9992e6","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:11.572Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"28c7161d-6431-4477-acdd-c6a9df29540c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:11.572Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"28c7161d-6431-4477-acdd-c6a9df29540c","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:11.572Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"28c7161d-6431-4477-acdd-c6a9df29540c","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:11.573Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"dafda1d9-aa98-497a-9004-7bf0ac9992e6","pod":"fail-k8s-job-txc8t-head-j9ptk","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-10T10:37:11.573Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"dafda1d9-aa98-497a-9004-7bf0ac9992e6","pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","ip":"10.132.0.48"}
{"level":"info","ts":"2026-06-10T10:37:11.585Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"bab163f6-48f7-4002-a25d-f83fd03d2213","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:11.586Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"dafda1d9-aa98-497a-9004-7bf0ac9992e6","certificate":"ray-head-cert-fail-k8s-job-txc8t","dnsNames":["fail-k8s-job-txc8t-head-svc","localhost","fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc","fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.28","10.132.0.48","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:37:11.599Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"28c7161d-6431-4477-acdd-c6a9df29540c"}
{"level":"info","ts":"2026-06-10T10:37:11.599Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"28c7161d-6431-4477-acdd-c6a9df29540c","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:11.599Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"dafda1d9-aa98-497a-9004-7bf0ac9992e6","certificate":"ray-worker-cert-fail-k8s-job-txc8t","dnsNames":["fail-k8s-job-txc8t-worker-svc","localhost","fail-k8s-job-txc8t-small-group","fail-k8s-job-txc8t-worker-svc.test-ns-k92wt.svc","fail-k8s-job-txc8t-worker-svc.test-ns-k92wt.svc.cluster.local","*.fail-k8s-job-txc8t-worker-svc.test-ns-k92wt.svc","*.fail-k8s-job-txc8t-worker-svc.test-ns-k92wt.svc.cluster.local","*-worker-*.test-ns-k92wt.svc","*-worker-*.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.28","10.132.0.48","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:37:11.599Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"dafda1d9-aa98-497a-9004-7bf0ac9992e6","rayCluster":"fail-k8s-job-txc8t","podIPs":["10.134.0.28","10.132.0.48"]}
{"level":"info","ts":"2026-06-10T10:37:11.604Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"bab163f6-48f7-4002-a25d-f83fd03d2213","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:11.604Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"bab163f6-48f7-4002-a25d-f83fd03d2213","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4"}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4"}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:12.410Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:12.411Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:12.411Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a7b68785-cc6e-47f8-9f64-23dcf16c9ae4","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:13.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bd8d44bf-4469-4044-a581-2104926eff5b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:13.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bd8d44bf-4469-4044-a581-2104926eff5b","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:13.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bd8d44bf-4469-4044-a581-2104926eff5b","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:13.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bd8d44bf-4469-4044-a581-2104926eff5b"}
{"level":"info","ts":"2026-06-10T10:37:13.586Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bd8d44bf-4469-4044-a581-2104926eff5b","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7"}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7"}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:16.580Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"1038abd1-ef04-4352-813c-0e380436b5f7","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:16.587Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"338102fb-da68-4b9e-8c1c-95488dbce0a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:16.587Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"338102fb-da68-4b9e-8c1c-95488dbce0a8","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:16.587Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"338102fb-da68-4b9e-8c1c-95488dbce0a8","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:16.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"338102fb-da68-4b9e-8c1c-95488dbce0a8"}
{"level":"info","ts":"2026-06-10T10:37:16.588Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"338102fb-da68-4b9e-8c1c-95488dbce0a8","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:17.584Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46"}
{"level":"info","ts":"2026-06-10T10:37:17.584Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46"}
{"level":"info","ts":"2026-06-10T10:37:17.584Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:37:17.585Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"f161f3b4-0732-4d87-81e9-beba652a6a46","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7e21426a-dd3d-4b10-9d41-ec41dcc86ad3","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"27fc922b-30e3-42c1-85ba-07aecba43b78","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"da56f5ad-8b3c-4400-b176-f9fc60a10683","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"da56f5ad-8b3c-4400-b176-f9fc60a10683","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"da56f5ad-8b3c-4400-b176-f9fc60a10683","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7e21426a-dd3d-4b10-9d41-ec41dcc86ad3","pod":"fail-k8s-job-txc8t-head-j9ptk","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7e21426a-dd3d-4b10-9d41-ec41dcc86ad3","pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","ip":"10.132.0.48"}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7e21426a-dd3d-4b10-9d41-ec41dcc86ad3","rayCluster":"fail-k8s-job-txc8t","podIPs":["10.134.0.28","10.132.0.48"]}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"da56f5ad-8b3c-4400-b176-f9fc60a10683"}
{"level":"info","ts":"2026-06-10T10:37:17.597Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"da56f5ad-8b3c-4400-b176-f9fc60a10683","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:17.606Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"27fc922b-30e3-42c1-85ba-07aecba43b78","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:17.617Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"27fc922b-30e3-42c1-85ba-07aecba43b78","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:17.617Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"27fc922b-30e3-42c1-85ba-07aecba43b78","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:19.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"66194829-2aea-4c34-868f-31538a269b10","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:19.588Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"66194829-2aea-4c34-868f-31538a269b10","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:19.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"66194829-2aea-4c34-868f-31538a269b10","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:19.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"66194829-2aea-4c34-868f-31538a269b10"}
{"level":"info","ts":"2026-06-10T10:37:19.589Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"66194829-2aea-4c34-868f-31538a269b10","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2"}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2"}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:19.598Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"00198450-c669-447b-a4a4-39d58ad2f5d2","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:21.054Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce"}
{"level":"info","ts":"2026-06-10T10:37:21.054Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce"}
{"level":"info","ts":"2026-06-10T10:37:21.054Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:21.055Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"6fcd94f3-7815-4bae-bfca-973b865939ce","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9"}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9"}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:22.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:22.064Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","podName":"fail-k8s-job-txc8t-small-group-worker-t78sr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:37:22.064Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","podName":"fail-k8s-job-txc8t-small-group-worker-t78sr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:37:22.064Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:37:22.064Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"462f0af5-a240-4805-aa24-9d2610c2cfa9","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:22.075Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d2a18e8c-9224-4578-800b-08c142a3a761","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:22.075Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"790a79a9-c0ef-4a21-ab41-72e31f15ace6","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:22.075Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4cce073-9bd5-42f1-85ee-e11c5baa8f0f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:22.076Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4cce073-9bd5-42f1-85ee-e11c5baa8f0f","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:22.076Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4cce073-9bd5-42f1-85ee-e11c5baa8f0f","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:22.076Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"790a79a9-c0ef-4a21-ab41-72e31f15ace6","pod":"fail-k8s-job-txc8t-head-j9ptk","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-10T10:37:22.076Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"790a79a9-c0ef-4a21-ab41-72e31f15ace6","pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","ip":"10.132.0.48"}
{"level":"info","ts":"2026-06-10T10:37:22.076Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"790a79a9-c0ef-4a21-ab41-72e31f15ace6","rayCluster":"fail-k8s-job-txc8t","podIPs":["10.134.0.28","10.132.0.48"]}
{"level":"info","ts":"2026-06-10T10:37:22.076Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4cce073-9bd5-42f1-85ee-e11c5baa8f0f"}
{"level":"info","ts":"2026-06-10T10:37:22.076Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4cce073-9bd5-42f1-85ee-e11c5baa8f0f","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:22.090Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d2a18e8c-9224-4578-800b-08c142a3a761","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:22.100Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d2a18e8c-9224-4578-800b-08c142a3a761","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:22.100Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d2a18e8c-9224-4578-800b-08c142a3a761","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:22.589Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"4fd052fb-13a4-483c-9782-c6d5b65db454","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:22.589Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"4fd052fb-13a4-483c-9782-c6d5b65db454","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:22.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"4fd052fb-13a4-483c-9782-c6d5b65db454","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:22.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"4fd052fb-13a4-483c-9782-c6d5b65db454"}
{"level":"info","ts":"2026-06-10T10:37:22.589Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"4fd052fb-13a4-483c-9782-c6d5b65db454","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:24.075Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c"}
{"level":"info","ts":"2026-06-10T10:37:24.075Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c"}
{"level":"info","ts":"2026-06-10T10:37:24.075Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:24.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:24.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:24.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:24.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:24.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:24.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:24.076Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:24.076Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:37:24.076Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","podName":"fail-k8s-job-txc8t-head-j9ptk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:37:24.076Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:24.076Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"10bc7734-ce6d-4914-9d0d-c79c5d7f9b5c","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:25.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c828ceae-a82c-46b3-a894-12b89ec9641f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:25.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c828ceae-a82c-46b3-a894-12b89ec9641f","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:25.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c828ceae-a82c-46b3-a894-12b89ec9641f","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:25.590Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c828ceae-a82c-46b3-a894-12b89ec9641f"}
{"level":"info","ts":"2026-06-10T10:37:25.590Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"c828ceae-a82c-46b3-a894-12b89ec9641f","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:28.591Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"6ac9864a-4a90-433e-9859-260c1b747750","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:28.591Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"6ac9864a-4a90-433e-9859-260c1b747750","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:28.591Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"6ac9864a-4a90-433e-9859-260c1b747750","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:28.591Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"6ac9864a-4a90-433e-9859-260c1b747750"}
{"level":"info","ts":"2026-06-10T10:37:28.591Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"6ac9864a-4a90-433e-9859-260c1b747750","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c"}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c"}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:28.615Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","podName":"fail-k8s-job-txc8t-small-group-worker-t78sr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:37:28.616Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","podName":"fail-k8s-job-txc8t-small-group-worker-t78sr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:37:28.616Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:37:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:37:28.616Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"a06479a8-bf5e-4523-824a-00eac9157a5c","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:37:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"9caa9d22-2ad1-4e6c-84e5-b3eef77e936c","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"1c1e1978-f045-49a3-83fa-13232aaa9e8b","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"9caa9d22-2ad1-4e6c-84e5-b3eef77e936c","pod":"fail-k8s-job-txc8t-head-j9ptk","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"9caa9d22-2ad1-4e6c-84e5-b3eef77e936c","pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","ip":"10.132.0.48"}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"9caa9d22-2ad1-4e6c-84e5-b3eef77e936c","rayCluster":"fail-k8s-job-txc8t","podIPs":["10.134.0.28","10.132.0.48"]}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9460aefa-b15a-4db1-bf65-30c17c03cf51","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9460aefa-b15a-4db1-bf65-30c17c03cf51","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9460aefa-b15a-4db1-bf65-30c17c03cf51","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9460aefa-b15a-4db1-bf65-30c17c03cf51"}
{"level":"info","ts":"2026-06-10T10:37:28.630Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9460aefa-b15a-4db1-bf65-30c17c03cf51","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:28.639Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"1c1e1978-f045-49a3-83fa-13232aaa9e8b","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:28.648Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"1c1e1978-f045-49a3-83fa-13232aaa9e8b","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:28.648Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"1c1e1978-f045-49a3-83fa-13232aaa9e8b","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13"}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13"}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","podName":"fail-k8s-job-txc8t-small-group-worker-t78sr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","podName":"fail-k8s-job-txc8t-small-group-worker-t78sr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:37:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:30.631Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"50e74451-7f3d-4298-8e98-38b842308f13","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:31.592Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed1cf7e2-d9fd-4182-b0de-8db2aa5e51be","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:31.592Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed1cf7e2-d9fd-4182-b0de-8db2aa5e51be","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:31.592Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed1cf7e2-d9fd-4182-b0de-8db2aa5e51be","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:31.592Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed1cf7e2-d9fd-4182-b0de-8db2aa5e51be"}
{"level":"info","ts":"2026-06-10T10:37:31.592Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ed1cf7e2-d9fd-4182-b0de-8db2aa5e51be","RayCluster":"fail-k8s-job-txc8t","State":""}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0"}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0"}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:37:33.093Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"090256cb-09e5-4698-9b44-95896ba8f3c0","name":"fail-k8s-job-txc8t","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:37:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:37:33Z","stateTransitionTimes":{"ready":"2026-06-10T10:37:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.28","serviceIP":"10.134.0.28","podName":"fail-k8s-job-txc8t-head-j9ptk","serviceName":"fail-k8s-job-txc8t-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:37:28Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:37:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:37:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:37:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:37:33.105Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2a60254a-6999-48b8-b3d1-5a4d096e9de7","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.105Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"feeb433a-c884-432e-92cb-8ca190a839dc","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.105Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2a60254a-6999-48b8-b3d1-5a4d096e9de7","pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","ip":"10.132.0.48"}
{"level":"info","ts":"2026-06-10T10:37:33.105Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2a60254a-6999-48b8-b3d1-5a4d096e9de7","pod":"fail-k8s-job-txc8t-head-j9ptk","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-10T10:37:33.105Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"2a60254a-6999-48b8-b3d1-5a4d096e9de7","rayCluster":"fail-k8s-job-txc8t","podIPs":["10.132.0.48","10.134.0.28"]}
{"level":"info","ts":"2026-06-10T10:37:33.105Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:33.105Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.105Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:33.106Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df"}
{"level":"info","ts":"2026-06-10T10:37:33.106Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","head service name":"fail-k8s-job-txc8t-head-svc","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:37:33.106Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","head service URL":"fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:37:33.106Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df"}
{"level":"info","ts":"2026-06-10T10:37:33.106Z","logger":"controllers.RayJob","msg":"User-provided command is used","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","command":["ray","job","submit","--address","http://do-not-exist:8265","--","echo 123"]}
{"level":"info","ts":"2026-06-10T10:37:33.114Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-06-10T10:37:33.114Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","SubmissionMode":"K8sJobMode","RayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.114Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-w6wqk","rayClusterName":"fail-k8s-job-txc8t","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:37:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-w6wqk","rayClusterName":"fail-k8s-job-txc8t","dashboardURL":"fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:37:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:37:33.114Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e1792ac2-fbc2-493f-a21d-2aa85e7c59df","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:37:33.118Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"feeb433a-c884-432e-92cb-8ca190a839dc","name":"fail-k8s-job-txc8t-head"}
{"level":"info","ts":"2026-06-10T10:37:33.125Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bdae00f0-6e31-4814-9514-351ad7a41d87","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:33.125Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bdae00f0-6e31-4814-9514-351ad7a41d87","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.125Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bdae00f0-6e31-4814-9514-351ad7a41d87","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:33.125Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bdae00f0-6e31-4814-9514-351ad7a41d87"}
{"level":"info","ts":"2026-06-10T10:37:33.133Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"feeb433a-c884-432e-92cb-8ca190a839dc","name":"fail-k8s-job-txc8t-workers"}
{"level":"info","ts":"2026-06-10T10:37:33.133Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"feeb433a-c884-432e-92cb-8ca190a839dc","cluster":"fail-k8s-job-txc8t"}
{"level":"error","ts":"2026-06-10T10:37:33.138Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bdae00f0-6e31-4814-9514-351ad7a41d87","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.138Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bdae00f0-6e31-4814-9514-351ad7a41d87"}
{"level":"error","ts":"2026-06-10T10:37:33.138Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bdae00f0-6e31-4814-9514-351ad7a41d87","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.139Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43242740-56cf-42b2-852c-98db7387b411","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:33.139Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43242740-56cf-42b2-852c-98db7387b411","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.139Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43242740-56cf-42b2-852c-98db7387b411","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:33.139Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43242740-56cf-42b2-852c-98db7387b411"}
{"level":"error","ts":"2026-06-10T10:37:33.144Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43242740-56cf-42b2-852c-98db7387b411","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.144Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43242740-56cf-42b2-852c-98db7387b411"}
{"level":"error","ts":"2026-06-10T10:37:33.144Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43242740-56cf-42b2-852c-98db7387b411","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.144Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"3148c808-76d0-4c90-b183-73afbc5da02b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:33.144Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"3148c808-76d0-4c90-b183-73afbc5da02b","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.144Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"3148c808-76d0-4c90-b183-73afbc5da02b","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:33.144Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"3148c808-76d0-4c90-b183-73afbc5da02b"}
{"level":"error","ts":"2026-06-10T10:37:33.149Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"3148c808-76d0-4c90-b183-73afbc5da02b","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.149Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"3148c808-76d0-4c90-b183-73afbc5da02b"}
{"level":"error","ts":"2026-06-10T10:37:33.149Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"3148c808-76d0-4c90-b183-73afbc5da02b","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.170Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43439375-f951-4b2a-afbb-7a3cc2240729","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:33.170Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43439375-f951-4b2a-afbb-7a3cc2240729","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.170Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43439375-f951-4b2a-afbb-7a3cc2240729","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:33.170Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43439375-f951-4b2a-afbb-7a3cc2240729"}
{"level":"error","ts":"2026-06-10T10:37:33.202Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43439375-f951-4b2a-afbb-7a3cc2240729","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.202Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43439375-f951-4b2a-afbb-7a3cc2240729"}
{"level":"error","ts":"2026-06-10T10:37:33.202Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"43439375-f951-4b2a-afbb-7a3cc2240729","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.243Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ffbd0ee6-e8b6-4fab-88f9-ca4a9eb1f72f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:33.243Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ffbd0ee6-e8b6-4fab-88f9-ca4a9eb1f72f","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.243Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ffbd0ee6-e8b6-4fab-88f9-ca4a9eb1f72f","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:33.243Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ffbd0ee6-e8b6-4fab-88f9-ca4a9eb1f72f"}
{"level":"error","ts":"2026-06-10T10:37:33.249Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ffbd0ee6-e8b6-4fab-88f9-ca4a9eb1f72f","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.249Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ffbd0ee6-e8b6-4fab-88f9-ca4a9eb1f72f"}
{"level":"error","ts":"2026-06-10T10:37:33.249Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"ffbd0ee6-e8b6-4fab-88f9-ca4a9eb1f72f","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.329Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"49bb526e-ea2d-407d-bad2-1313a77ba4ca","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:33.329Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"49bb526e-ea2d-407d-bad2-1313a77ba4ca","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.329Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"49bb526e-ea2d-407d-bad2-1313a77ba4ca","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:33.329Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"49bb526e-ea2d-407d-bad2-1313a77ba4ca"}
{"level":"error","ts":"2026-06-10T10:37:33.335Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"49bb526e-ea2d-407d-bad2-1313a77ba4ca","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.335Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"49bb526e-ea2d-407d-bad2-1313a77ba4ca"}
{"level":"error","ts":"2026-06-10T10:37:33.335Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"49bb526e-ea2d-407d-bad2-1313a77ba4ca","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.495Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bc5df7b9-1f63-4eef-8938-452a3f04a04d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:33.495Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bc5df7b9-1f63-4eef-8938-452a3f04a04d","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.495Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bc5df7b9-1f63-4eef-8938-452a3f04a04d","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:33.496Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bc5df7b9-1f63-4eef-8938-452a3f04a04d"}
{"level":"error","ts":"2026-06-10T10:37:33.502Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bc5df7b9-1f63-4eef-8938-452a3f04a04d","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.502Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bc5df7b9-1f63-4eef-8938-452a3f04a04d"}
{"level":"error","ts":"2026-06-10T10:37:33.502Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bc5df7b9-1f63-4eef-8938-452a3f04a04d","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.823Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"521764fa-855d-4c5f-824c-0c1bade8ab48","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:33.823Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"521764fa-855d-4c5f-824c-0c1bade8ab48","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:33.823Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"521764fa-855d-4c5f-824c-0c1bade8ab48","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:33.823Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"521764fa-855d-4c5f-824c-0c1bade8ab48"}
{"level":"error","ts":"2026-06-10T10:37:33.828Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"521764fa-855d-4c5f-824c-0c1bade8ab48","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:33.828Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"521764fa-855d-4c5f-824c-0c1bade8ab48"}
{"level":"error","ts":"2026-06-10T10:37:33.828Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"521764fa-855d-4c5f-824c-0c1bade8ab48","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:34.469Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"8d0995eb-2f63-4b03-b822-bd0d8532b2b4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:34.469Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"8d0995eb-2f63-4b03-b822-bd0d8532b2b4","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:34.469Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"8d0995eb-2f63-4b03-b822-bd0d8532b2b4","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:34.470Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"8d0995eb-2f63-4b03-b822-bd0d8532b2b4"}
{"level":"error","ts":"2026-06-10T10:37:34.475Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"8d0995eb-2f63-4b03-b822-bd0d8532b2b4","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:34.475Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"8d0995eb-2f63-4b03-b822-bd0d8532b2b4"}
{"level":"error","ts":"2026-06-10T10:37:34.475Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"8d0995eb-2f63-4b03-b822-bd0d8532b2b4","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1"}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1"}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","1 head service found":"fail-k8s-job-txc8t-head-svc"}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","Found 1 head Pod":"fail-k8s-job-txc8t-head-j9ptk","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","head Pod":"fail-k8s-job-txc8t-head-j9ptk","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-txc8t-head-j9ptk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","worker Pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-txc8t-small-group-worker-t78sr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:37:35.106Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"ab8d8635-519c-42e4-b0ce-430cba9642a1","seconds":300}
{"level":"info","ts":"2026-06-10T10:37:35.674Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"cf4f11f9-3762-450a-888b-10c30b5c61bd","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:35.674Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"cf4f11f9-3762-450a-888b-10c30b5c61bd","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:35.674Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"cf4f11f9-3762-450a-888b-10c30b5c61bd","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:35.674Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"cf4f11f9-3762-450a-888b-10c30b5c61bd"}
{"level":"error","ts":"2026-06-10T10:37:35.680Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"cf4f11f9-3762-450a-888b-10c30b5c61bd","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:35.680Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"cf4f11f9-3762-450a-888b-10c30b5c61bd"}
{"level":"error","ts":"2026-06-10T10:37:35.680Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"cf4f11f9-3762-450a-888b-10c30b5c61bd","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:35.756Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"94b68a27-44be-4417-8fab-6fd6092efd1c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:35.756Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"94b68a27-44be-4417-8fab-6fd6092efd1c","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:35.756Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"94b68a27-44be-4417-8fab-6fd6092efd1c","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:35.756Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"94b68a27-44be-4417-8fab-6fd6092efd1c"}
{"level":"error","ts":"2026-06-10T10:37:35.762Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"94b68a27-44be-4417-8fab-6fd6092efd1c","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:35.762Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"94b68a27-44be-4417-8fab-6fd6092efd1c"}
{"level":"error","ts":"2026-06-10T10:37:35.762Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"94b68a27-44be-4417-8fab-6fd6092efd1c","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:37.825Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"a4b53423-4076-4834-9446-970ea3adebe2","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:37.825Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"a4b53423-4076-4834-9446-970ea3adebe2","pod":"fail-k8s-job-txc8t-head-j9ptk","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-10T10:37:37.825Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"a4b53423-4076-4834-9446-970ea3adebe2","pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","ip":"10.132.0.48"}
{"level":"info","ts":"2026-06-10T10:37:37.825Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"a4b53423-4076-4834-9446-970ea3adebe2","rayCluster":"fail-k8s-job-txc8t","podIPs":["10.134.0.28","10.132.0.48"]}
{"level":"info","ts":"2026-06-10T10:37:38.686Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e00d188c-12bd-4921-a4d3-de2d78c5a80d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:38.687Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e00d188c-12bd-4921-a4d3-de2d78c5a80d","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:38.687Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e00d188c-12bd-4921-a4d3-de2d78c5a80d","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:38.687Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e00d188c-12bd-4921-a4d3-de2d78c5a80d"}
{"level":"error","ts":"2026-06-10T10:37:38.693Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e00d188c-12bd-4921-a4d3-de2d78c5a80d","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:38.693Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e00d188c-12bd-4921-a4d3-de2d78c5a80d"}
{"level":"error","ts":"2026-06-10T10:37:38.693Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e00d188c-12bd-4921-a4d3-de2d78c5a80d","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:39.695Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"15bf3537-e3e8-4bed-ad9e-0feb3b9cc93c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:39.695Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"15bf3537-e3e8-4bed-ad9e-0feb3b9cc93c","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:39.695Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"15bf3537-e3e8-4bed-ad9e-0feb3b9cc93c","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:39.695Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"15bf3537-e3e8-4bed-ad9e-0feb3b9cc93c"}
{"level":"error","ts":"2026-06-10T10:37:39.700Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"15bf3537-e3e8-4bed-ad9e-0feb3b9cc93c","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:39.700Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"15bf3537-e3e8-4bed-ad9e-0feb3b9cc93c"}
{"level":"error","ts":"2026-06-10T10:37:39.700Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"15bf3537-e3e8-4bed-ad9e-0feb3b9cc93c","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:39.710Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"1bd7428e-cc3a-45d0-8ebb-deb5a5750ea0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:39.710Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"1bd7428e-cc3a-45d0-8ebb-deb5a5750ea0","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:39.710Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"1bd7428e-cc3a-45d0-8ebb-deb5a5750ea0","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:39.710Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"1bd7428e-cc3a-45d0-8ebb-deb5a5750ea0"}
{"level":"error","ts":"2026-06-10T10:37:39.714Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"1bd7428e-cc3a-45d0-8ebb-deb5a5750ea0","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:39.715Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"1bd7428e-cc3a-45d0-8ebb-deb5a5750ea0"}
{"level":"error","ts":"2026-06-10T10:37:39.715Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"1bd7428e-cc3a-45d0-8ebb-deb5a5750ea0","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:40.883Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e59ecd3c-25e5-4c07-b4ff-e0a352518b8f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:40.883Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e59ecd3c-25e5-4c07-b4ff-e0a352518b8f","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:40.883Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e59ecd3c-25e5-4c07-b4ff-e0a352518b8f","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:40.883Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e59ecd3c-25e5-4c07-b4ff-e0a352518b8f"}
{"level":"error","ts":"2026-06-10T10:37:40.890Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e59ecd3c-25e5-4c07-b4ff-e0a352518b8f","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:40.890Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e59ecd3c-25e5-4c07-b4ff-e0a352518b8f"}
{"level":"error","ts":"2026-06-10T10:37:40.890Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"e59ecd3c-25e5-4c07-b4ff-e0a352518b8f","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:46.914Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7ddf0efc-15ca-4592-bb8a-7be7c0decac0","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:37:46.915Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7ddf0efc-15ca-4592-bb8a-7be7c0decac0","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:37:46.915Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7ddf0efc-15ca-4592-bb8a-7be7c0decac0","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:37:46.915Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"7ddf0efc-15ca-4592-bb8a-7be7c0decac0","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:37:47.032Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7bc908fc-4472-41f4-b6a3-7487a4da9d39","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:47.032Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7bc908fc-4472-41f4-b6a3-7487a4da9d39","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:47.032Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7bc908fc-4472-41f4-b6a3-7487a4da9d39","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:47.032Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7bc908fc-4472-41f4-b6a3-7487a4da9d39"}
{"level":"error","ts":"2026-06-10T10:37:47.039Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7bc908fc-4472-41f4-b6a3-7487a4da9d39","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:47.039Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7bc908fc-4472-41f4-b6a3-7487a4da9d39"}
{"level":"error","ts":"2026-06-10T10:37:47.039Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7bc908fc-4472-41f4-b6a3-7487a4da9d39","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:48.027Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bab4a06d-6503-43d7-8ca3-4fd159379031","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:48.027Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bab4a06d-6503-43d7-8ca3-4fd159379031","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:48.027Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bab4a06d-6503-43d7-8ca3-4fd159379031","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:48.027Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bab4a06d-6503-43d7-8ca3-4fd159379031"}
{"level":"error","ts":"2026-06-10T10:37:48.032Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bab4a06d-6503-43d7-8ca3-4fd159379031","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:48.032Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bab4a06d-6503-43d7-8ca3-4fd159379031"}
{"level":"error","ts":"2026-06-10T10:37:48.032Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"bab4a06d-6503-43d7-8ca3-4fd159379031","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:52.755Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"63632edf-01bd-439d-806d-d9e416646711","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:52.755Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"63632edf-01bd-439d-806d-d9e416646711","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:52.755Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"63632edf-01bd-439d-806d-d9e416646711","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:52.755Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"63632edf-01bd-439d-806d-d9e416646711"}
{"level":"error","ts":"2026-06-10T10:37:52.761Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"63632edf-01bd-439d-806d-d9e416646711","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:52.761Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"63632edf-01bd-439d-806d-d9e416646711"}
{"level":"error","ts":"2026-06-10T10:37:52.761Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"63632edf-01bd-439d-806d-d9e416646711","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:53.764Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f6195460-9c6f-4d7b-97fe-be360689878d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:53.764Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f6195460-9c6f-4d7b-97fe-be360689878d","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:53.764Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f6195460-9c6f-4d7b-97fe-be360689878d","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:53.764Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f6195460-9c6f-4d7b-97fe-be360689878d"}
{"level":"error","ts":"2026-06-10T10:37:53.769Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f6195460-9c6f-4d7b-97fe-be360689878d","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:53.770Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f6195460-9c6f-4d7b-97fe-be360689878d"}
{"level":"error","ts":"2026-06-10T10:37:53.770Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f6195460-9c6f-4d7b-97fe-be360689878d","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:53.784Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"440cdb07-f777-4856-83dd-69b769fe2153","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:37:53.784Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"440cdb07-f777-4856-83dd-69b769fe2153","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:37:53.784Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"440cdb07-f777-4856-83dd-69b769fe2153","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:37:53.784Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"440cdb07-f777-4856-83dd-69b769fe2153"}
{"level":"error","ts":"2026-06-10T10:37:53.789Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"440cdb07-f777-4856-83dd-69b769fe2153","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:37:53.789Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"440cdb07-f777-4856-83dd-69b769fe2153"}
{"level":"error","ts":"2026-06-10T10:37:53.789Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"440cdb07-f777-4856-83dd-69b769fe2153","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:11.019Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"850a08ec-b201-43b8-8d4c-be641c1e1cce","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:11.019Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"850a08ec-b201-43b8-8d4c-be641c1e1cce","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:11.019Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"850a08ec-b201-43b8-8d4c-be641c1e1cce","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:11.019Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"850a08ec-b201-43b8-8d4c-be641c1e1cce"}
{"level":"error","ts":"2026-06-10T10:38:11.029Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"850a08ec-b201-43b8-8d4c-be641c1e1cce","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:11.030Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"850a08ec-b201-43b8-8d4c-be641c1e1cce"}
{"level":"error","ts":"2026-06-10T10:38:11.030Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"850a08ec-b201-43b8-8d4c-be641c1e1cce","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:12.021Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9f355462-2e60-4f24-a96a-4dd0eae79e77","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:12.021Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9f355462-2e60-4f24-a96a-4dd0eae79e77","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:12.021Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9f355462-2e60-4f24-a96a-4dd0eae79e77","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:12.021Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9f355462-2e60-4f24-a96a-4dd0eae79e77"}
{"level":"error","ts":"2026-06-10T10:38:12.026Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9f355462-2e60-4f24-a96a-4dd0eae79e77","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:12.026Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9f355462-2e60-4f24-a96a-4dd0eae79e77"}
{"level":"error","ts":"2026-06-10T10:38:12.026Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"9f355462-2e60-4f24-a96a-4dd0eae79e77","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:16.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"d532ecdb-285a-4077-93ac-cea760e95e45","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:16.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"d532ecdb-285a-4077-93ac-cea760e95e45","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:16.847Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"d532ecdb-285a-4077-93ac-cea760e95e45","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:16.848Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"d532ecdb-285a-4077-93ac-cea760e95e45"}
{"level":"error","ts":"2026-06-10T10:38:16.854Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"d532ecdb-285a-4077-93ac-cea760e95e45","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:16.854Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"d532ecdb-285a-4077-93ac-cea760e95e45"}
{"level":"error","ts":"2026-06-10T10:38:16.854Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"d532ecdb-285a-4077-93ac-cea760e95e45","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:17.853Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4507971-871c-423b-9a4b-444775371990","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:17.853Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4507971-871c-423b-9a4b-444775371990","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.853Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4507971-871c-423b-9a4b-444775371990","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:17.853Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4507971-871c-423b-9a4b-444775371990"}
{"level":"error","ts":"2026-06-10T10:38:17.859Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4507971-871c-423b-9a4b-444775371990","JobId":"fail-k8s-job-w6wqk","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:17.859Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4507971-871c-423b-9a4b-444775371990"}
{"level":"error","ts":"2026-06-10T10:38:17.859Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"b4507971-871c-423b-9a4b-444775371990","error":"Job fail-k8s-job-w6wqk does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:17.872Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"50c79dc2-74d2-453c-a16e-e55fb8c6c139","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:17.872Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has failed. Attempting to transition the status to `Failed`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"50c79dc2-74d2-453c-a16e-e55fb8c6c139","Submitter K8s Job":"fail-k8s-job","Reason":"BackoffLimitExceeded","Message":"Job has reached the specified backoff limit"}
{"level":"info","ts":"2026-06-10T10:38:17.872Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"50c79dc2-74d2-453c-a16e-e55fb8c6c139","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-w6wqk","rayClusterName":"fail-k8s-job-txc8t","dashboardURL":"fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:37:07Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-w6wqk","rayClusterName":"fail-k8s-job-txc8t","dashboardURL":"fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"SubmissionFailed","message":"Job submission has failed. Reason: BackoffLimitExceeded. Message: Job has reached the specified backoff limit","startTime":"2026-06-10T10:37:07Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:38:17.872Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"50c79dc2-74d2-453c-a16e-e55fb8c6c139","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-10T10:38:17.886Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7462f3fb-4612-45ad-aae0-bcdafde55d2a","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:17.886Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7462f3fb-4612-45ad-aae0-bcdafde55d2a","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:38:17 +0000 UTC","Now":"2026-06-10T10:38:17.886Z","ShutdownTime":"2026-06-10T10:38:17.000Z"}
{"level":"info","ts":"2026-06-10T10:38:17.886Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7462f3fb-4612-45ad-aae0-bcdafde55d2a","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"reconcileID":"220d1e68-c4fe-431b-9b3a-039f66f5327f"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"01e6b46b-3691-4844-a188-4309006716a1","namespacedName":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"f3f17c8b-93ae-46bc-8e78-a0e7f5b20528","rayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"01e6b46b-3691-4844-a188-4309006716a1","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"b9221ec0-0dbc-49d9-b959-74b40b68e821"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"01e6b46b-3691-4844-a188-4309006716a1","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"01e6b46b-3691-4844-a188-4309006716a1","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"f3f17c8b-93ae-46bc-8e78-a0e7f5b20528","pod":"fail-k8s-job-txc8t-small-group-worker-t78sr","ip":"10.132.0.48"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"f3f17c8b-93ae-46bc-8e78-a0e7f5b20528","pod":"fail-k8s-job-txc8t-head-j9ptk","ip":"10.134.0.28"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7462f3fb-4612-45ad-aae0-bcdafde55d2a","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"f3f17c8b-93ae-46bc-8e78-a0e7f5b20528","rayCluster":"fail-k8s-job-txc8t","podIPs":["10.132.0.48","10.134.0.28"]}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7462f3fb-4612-45ad-aae0-bcdafde55d2a","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"7462f3fb-4612-45ad-aae0-bcdafde55d2a","RayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"abc21b96-4356-4566-86fe-041cd7613912","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"abc21b96-4356-4566-86fe-041cd7613912","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:38:17 +0000 UTC","Now":"2026-06-10T10:38:17.898Z","ShutdownTime":"2026-06-10T10:38:17.000Z"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"abc21b96-4356-4566-86fe-041cd7613912","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"abc21b96-4356-4566-86fe-041cd7613912","RayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"abc21b96-4356-4566-86fe-041cd7613912","isClusterDeleted":false}
{"level":"info","ts":"2026-06-10T10:38:17.898Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"abc21b96-4356-4566-86fe-041cd7613912","RayCluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.907Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"01e6b46b-3691-4844-a188-4309006716a1","name":"kuberay-gateway-access","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:38:17.912Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"01e6b46b-3691-4844-a188-4309006716a1","name":"test-ns-k92wt-fail-k8s-job-txc8t","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:38:17.917Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"01e6b46b-3691-4844-a188-4309006716a1","configMap":"kube-rbac-proxy-config-fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.922Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"01e6b46b-3691-4844-a188-4309006716a1","serviceAccount":"fail-k8s-job-txc8t-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:38:17.931Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.931Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:38:17.943Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"01e6b46b-3691-4844-a188-4309006716a1","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.943Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"4ada2664-a902-4fff-b1de-1f303e0eb433","namespacedName":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:17.943Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"4ada2664-a902-4fff-b1de-1f303e0eb433","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.943Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"4ada2664-a902-4fff-b1de-1f303e0eb433","cluster":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.943Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"4ada2664-a902-4fff-b1de-1f303e0eb433","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:38:17.943Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"4ada2664-a902-4fff-b1de-1f303e0eb433","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:38:17.944Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"1aa02e59-030a-474f-81b1-5e3cddb6fff3"}
{"level":"info","ts":"2026-06-10T10:38:17.944Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"75d81809-422d-4b47-b48b-4453ce5f80f7","namespace":"test-ns-k92wt","clusterName":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:17.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f85360c8-4aa9-4d07-a9c1-5d3b7032664e","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:17.944Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f85360c8-4aa9-4d07-a9c1-5d3b7032664e","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:38:17 +0000 UTC","Now":"2026-06-10T10:38:17.944Z","ShutdownTime":"2026-06-10T10:38:17.000Z"}
{"level":"info","ts":"2026-06-10T10:38:17.944Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f85360c8-4aa9-4d07-a9c1-5d3b7032664e","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:38:17.944Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f85360c8-4aa9-4d07-a9c1-5d3b7032664e","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:17.944Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f85360c8-4aa9-4d07-a9c1-5d3b7032664e","isClusterDeleted":true}
{"level":"info","ts":"2026-06-10T10:38:17.944Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"f85360c8-4aa9-4d07-a9c1-5d3b7032664e","RayCluster":"fail-k8s-job-txc8t"}
{"level":"error","ts":"2026-06-10T10:38:17.947Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"4ada2664-a902-4fff-b1de-1f303e0eb433","error":"rayclusters.ray.io \"fail-k8s-job-txc8t\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:17.947Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"4ada2664-a902-4fff-b1de-1f303e0eb433"}
{"level":"error","ts":"2026-06-10T10:38:17.947Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"4ada2664-a902-4fff-b1de-1f303e0eb433","error":"rayclusters.ray.io \"fail-k8s-job-txc8t\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:17.947Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ae21d0e8-1fe1-4b31-947c-2bad504c2ea5","namespacedName":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:17.947Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ae21d0e8-1fe1-4b31-947c-2bad504c2ea5"}
{"level":"info","ts":"2026-06-10T10:38:17.947Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ae21d0e8-1fe1-4b31-947c-2bad504c2ea5","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:38:17.947Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ae21d0e8-1fe1-4b31-947c-2bad504c2ea5","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:38:17.953Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"6c69ca33-00d2-4f08-8a6e-dcc643166119","namespacedName":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:17.953Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"6c69ca33-00d2-4f08-8a6e-dcc643166119"}
{"level":"info","ts":"2026-06-10T10:38:17.953Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"6c69ca33-00d2-4f08-8a6e-dcc643166119","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:38:17.953Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"6c69ca33-00d2-4f08-8a6e-dcc643166119","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:38:17.954Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"ad50101b-856a-456f-bda3-240caa9670f9"}
{"level":"error","ts":"2026-06-10T10:38:17.955Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"75d81809-422d-4b47-b48b-4453ce5f80f7","issuer":"ray-selfsigned-issuer-fail-k8s-job-txc8t","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-k8s-job-txc8t\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:38:17.958Z","msg":"Failed to delete head certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"75d81809-422d-4b47-b48b-4453ce5f80f7","certificate":"ray-head-cert-fail-k8s-job-txc8t","error":"certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-txc8t\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:261\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:17.962Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7f7a45ed-a1d0-452a-a124-5b8a76a5d99b","namespacedName":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:17.962Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7f7a45ed-a1d0-452a-a124-5b8a76a5d99b"}
{"level":"info","ts":"2026-06-10T10:38:17.962Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7f7a45ed-a1d0-452a-a124-5b8a76a5d99b","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:38:17.962Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"7f7a45ed-a1d0-452a-a124-5b8a76a5d99b","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:38:17.965Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"d35494ea-da91-477b-aa2b-8e115863ee7b"}
{"level":"error","ts":"2026-06-10T10:38:17.971Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"75d81809-422d-4b47-b48b-4453ce5f80f7","certificate":"ray-worker-cert-fail-k8s-job-txc8t","error":"certificates.cert-manager.io \"ray-worker-cert-fail-k8s-job-txc8t\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.006Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"75d81809-422d-4b47-b48b-4453ce5f80f7","errorCount":3}
{"level":"info","ts":"2026-06-10T10:38:18.006Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"75d81809-422d-4b47-b48b-4453ce5f80f7"}
{"level":"error","ts":"2026-06-10T10:38:18.006Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"75d81809-422d-4b47-b48b-4453ce5f80f7","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-fail-k8s-job-txc8t\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.012Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"fd48751f-74e3-4b45-a3e8-6b1961afa6cc","namespace":"test-ns-k92wt","clusterName":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:18.012Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-txc8t","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"fail-k8s-job-txc8t","reconcileID":"fd48751f-74e3-4b45-a3e8-6b1961afa6cc","namespace":"test-ns-k92wt","clusterName":"fail-k8s-job-txc8t"}
{"level":"info","ts":"2026-06-10T10:38:18.023Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"56a0c973-cb01-42e7-b678-70c5e86deacc","DeletionTimestamp":"2026-06-10 10:38:18 +0000 UTC"}
{"level":"error","ts":"2026-06-10T10:38:18.023Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"56a0c973-cb01-42e7-b678-70c5e86deacc","error":"RayCluster.ray.io \"fail-k8s-job-txc8t\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.023Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"56a0c973-cb01-42e7-b678-70c5e86deacc","rayJob":"fail-k8s-job-w6wqk"}
{"level":"error","ts":"2026-06-10T10:38:18.061Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"56a0c973-cb01-42e7-b678-70c5e86deacc","error":"Post \"http://fail-k8s-job-txc8t-head-svc.test-ns-k92wt.svc.cluster.local:8265/api/jobs/fail-k8s-job-w6wqk/stop\": EOF","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.061Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"56a0c973-cb01-42e7-b678-70c5e86deacc","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:38:18.073Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"38ba089b-b45b-4b12-b40e-6b0aeaa88174","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:18.073Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"38ba089b-b45b-4b12-b40e-6b0aeaa88174","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:38:18.084Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"38ba089b-b45b-4b12-b40e-6b0aeaa88174"}
{"level":"info","ts":"2026-06-10T10:38:18.084Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"38ba089b-b45b-4b12-b40e-6b0aeaa88174","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:38:18.084Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"38ba089b-b45b-4b12-b40e-6b0aeaa88174","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:38:18.084Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"38ba089b-b45b-4b12-b40e-6b0aeaa88174","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:38:18.099Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"a8112d53-4792-407e-b14a-3d609e4811d9"}
{"level":"info","ts":"2026-06-10T10:38:18.099Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"046172fb-b50e-462e-9fa2-bf88f90fe1ea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:18.099Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"046172fb-b50e-462e-9fa2-bf88f90fe1ea","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.099Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"046172fb-b50e-462e-9fa2-bf88f90fe1ea","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:18.099Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"046172fb-b50e-462e-9fa2-bf88f90fe1ea","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:18.104Z","logger":"raycluster-resource","msg":"default","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.104Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-kp6pp","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"046172fb-b50e-462e-9fa2-bf88f90fe1ea","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","namespacedName":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"046172fb-b50e-462e-9fa2-bf88f90fe1ea"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","mode":"IntegratedOAuth","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"046172fb-b50e-462e-9fa2-bf88f90fe1ea","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"fe749719-71b7-4f22-a878-cd83cd0e75d3","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"edc7c8f3-d5c5-4b39-9553-0c14f2f6dc4d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:18.118Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"edc7c8f3-d5c5-4b39-9553-0c14f2f6dc4d","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.119Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"edc7c8f3-d5c5-4b39-9553-0c14f2f6dc4d","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:18.119Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"edc7c8f3-d5c5-4b39-9553-0c14f2f6dc4d"}
{"level":"info","ts":"2026-06-10T10:38:18.119Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"edc7c8f3-d5c5-4b39-9553-0c14f2f6dc4d","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:18.124Z","logger":"raycluster-resource","msg":"default","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.124Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-kp6pp","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:38:18.129Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","name":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:18.129Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","error":"head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"error","ts":"2026-06-10T10:38:18.129Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.129Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:18.129Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.129Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.129Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.129Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:38:18.129Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","name":"stop-kp6pp","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:18.130Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.130Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","name":"stop-kp6pp","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-kp6pp\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"stop-kp6pp"}}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"fe749719-71b7-4f22-a878-cd83cd0e75d3","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"83ca24f9-faf3-494b-85a9-a8b5b4f82bbb","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6"}
{"level":"error","ts":"2026-06-10T10:38:18.199Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"103760fb-8cfd-46f0-a3bf-47bd931022d6","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8","rayCluster":"stop-kp6pp","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:18.199Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","error":"head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"error","ts":"2026-06-10T10:38:18.199Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.200Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:18.200Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f7aae041-b366-4168-82bd-ff4018a406f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:18.200Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f7aae041-b366-4168-82bd-ff4018a406f3","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.200Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f7aae041-b366-4168-82bd-ff4018a406f3","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:18.200Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.200Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.200Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.200Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:38:18.200Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","name":"stop-kp6pp","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:18.206Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","name":"stop-kp6pp-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a"}
{"level":"error","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"7f6fad49-b744-4974-b618-4aa9f313c92a","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168"}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168"}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","error":"head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"error","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:38:18.210Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","name":"stop-kp6pp","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:18.212Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"83ca24f9-faf3-494b-85a9-a8b5b4f82bbb","name":"stop-kp6pp-head"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f7aae041-b366-4168-82bd-ff4018a406f3"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f7aae041-b366-4168-82bd-ff4018a406f3","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","name":"kuberay-gateway-access","namespace":"test-ns-k92wt","operation":"created"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3c87037b-d4d7-43a2-b654-a1a68b790406","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3c87037b-d4d7-43a2-b654-a1a68b790406","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3c87037b-d4d7-43a2-b654-a1a68b790406","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","name":"stop-kp6pp","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-kp6pp\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"stop-kp6pp"}}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3c87037b-d4d7-43a2-b654-a1a68b790406"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3c87037b-d4d7-43a2-b654-a1a68b790406","RayCluster":"stop-kp6pp","State":""}
{"level":"error","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4bcd6823-3fd9-4136-854f-9f2f5c519168","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396","error":"head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"error","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396"}
{"level":"error","ts":"2026-06-10T10:38:18.299Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"4aac4d5f-4094-40d0-b776-e2b6f9a12396","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.300Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.303Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"83ca24f9-faf3-494b-85a9-a8b5b4f82bbb","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:18.303Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"83ca24f9-faf3-494b-85a9-a8b5b4f82bbb","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.303Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"5f8b0b3d-03de-48e9-a6e2-3c24e35e5aef","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.308Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","name":"test-ns-k92wt-stop-kp6pp","operation":"created"}
{"level":"info","ts":"2026-06-10T10:38:18.311Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8"}
{"level":"info","ts":"2026-06-10T10:38:18.311Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8","rayCluster":"stop-kp6pp","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:38:18.313Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","name":"kube-rbac-proxy-config-stop-kp6pp","operation":"created"}
{"level":"info","ts":"2026-06-10T10:38:18.315Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"5f8b0b3d-03de-48e9-a6e2-3c24e35e5aef","name":"stop-kp6pp-head"}
{"level":"error","ts":"2026-06-10T10:38:18.319Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-kp6pp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e"}
{"level":"error","ts":"2026-06-10T10:38:18.319Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c2b78f7e-75d5-4102-af26-974413a15d9e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-kp6pp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"a75b9302-98de-40f0-8052-3ab16dd85209","namespacedName":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"a75b9302-98de-40f0-8052-3ab16dd85209","mode":"IntegratedOAuth","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"a75b9302-98de-40f0-8052-3ab16dd85209","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8"}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8","rayCluster":"stop-kp6pp","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"ab25c3ae-248c-4a51-8762-2e639330a2d8"}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"f7af2209-1056-4536-9151-a8668d0edb4c","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"f7af2209-1056-4536-9151-a8668d0edb4c"}
{"level":"info","ts":"2026-06-10T10:38:18.319Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179"}
{"level":"info","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179"}
{"level":"info","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179","error":"head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"error","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179"}
{"level":"error","ts":"2026-06-10T10:38:18.320Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"14355703-e563-406e-88c4-a14a6f8fd179","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"a75b9302-98de-40f0-8052-3ab16dd85209","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"da5c4b81-59eb-4197-b4ed-e0eb498d6ac6","namespacedName":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c24ceaf4-a8b5-4460-9b07-5bd3b8335711","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"da5c4b81-59eb-4197-b4ed-e0eb498d6ac6","mode":"IntegratedOAuth","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"da5c4b81-59eb-4197-b4ed-e0eb498d6ac6","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c24ceaf4-a8b5-4460-9b07-5bd3b8335711"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3e396839-679e-4a3c-9a65-648724dd4d9f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3e396839-679e-4a3c-9a65-648724dd4d9f","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2","error":"head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3e396839-679e-4a3c-9a65-648724dd4d9f","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"error","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3e396839-679e-4a3c-9a65-648724dd4d9f"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2"}
{"level":"info","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"3e396839-679e-4a3c-9a65-648724dd4d9f","RayCluster":"stop-kp6pp","State":""}
{"level":"error","ts":"2026-06-10T10:38:18.400Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"02d61004-748e-46b2-9ee6-5d24fa1a72b2","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.405Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"5f8b0b3d-03de-48e9-a6e2-3c24e35e5aef","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:18.405Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"5f8b0b3d-03de-48e9-a6e2-3c24e35e5aef","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.405Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"400117c6-1e5d-483f-a04e-e9c92b423f1d","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.409Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"da5c4b81-59eb-4197-b4ed-e0eb498d6ac6","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.416Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"400117c6-1e5d-483f-a04e-e9c92b423f1d","name":"stop-kp6pp-head"}
{"level":"info","ts":"2026-06-10T10:38:18.499Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"400117c6-1e5d-483f-a04e-e9c92b423f1d","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:18.499Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"400117c6-1e5d-483f-a04e-e9c92b423f1d","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa"}
{"level":"info","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa"}
{"level":"info","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa","error":"head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"error","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa"}
{"level":"error","ts":"2026-06-10T10:38:18.561Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"21625e40-0708-4919-9d19-1a4aea14a6fa","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1"}
{"level":"info","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1"}
{"level":"info","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1","error":"head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"error","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1"}
{"level":"error","ts":"2026-06-10T10:38:18.882Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dcac8c1d-c7cc-4d1b-a286-66f5107f88f1","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700"}
{"level":"info","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700"}
{"level":"info","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700","error":"head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"error","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found"}
{"level":"info","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700","filter labels":["test-ns-k92wt",{"ray.io/cluster":"stop-kp6pp","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700"}
{"level":"error","ts":"2026-06-10T10:38:19.523Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"88b3d3ed-17cf-40f6-b827-e75c5f4e2700","error":"mTLS secrets not ready: head certificate secret ray-head-secret-stop-kp6pp not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f"}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f"}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f"}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f"}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","mode":"IntegratedOAuth","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","cluster":"stop-kp6pp","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","cluster":"stop-kp6pp","authType":"IntegratedOAuth","serviceAccount":"stop-kp6pp-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-kp6pp","ray.io/group":"headgroup","ray.io/identifier":"stop-kp6pp-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:38:20.804Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","enabled":true}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","name":"stop-kp6pp-head-7s9fj"}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f"}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","nodeType":"worker","rayStartParams":{"address":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","rayStartCmd":"ray start  --address=stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:38:20.820Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","enabled":true}
{"level":"info","ts":"2026-06-10T10:38:20.832Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","name":"stop-kp6pp-small-group-worker-5sk92"}
{"level":"info","ts":"2026-06-10T10:38:20.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.833Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-kp6pp-head-svc"},"newHead":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"}}
{"level":"info","ts":"2026-06-10T10:38:20.833Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f2d964ec-da6c-4a9b-9cd6-9b03b970ef8f","name":"stop-kp6pp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb"}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb"}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"stop-kp6pp-head-svc"},"newHead":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"}}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","name":"stop-kp6pp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:20.847Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"3b3da0b1-fdc7-4bc4-90fa-a4e23b5dbc12","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:20.848Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"57b3101e-1c64-494b-92e6-f8abefcd0ff5","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:20.848Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6ce4ea36-4fa9-4520-b5ac-ebd6a8bcc206","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:20.848Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6ce4ea36-4fa9-4520-b5ac-ebd6a8bcc206","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:20.848Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6ce4ea36-4fa9-4520-b5ac-ebd6a8bcc206","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:20.848Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6ce4ea36-4fa9-4520-b5ac-ebd6a8bcc206"}
{"level":"info","ts":"2026-06-10T10:38:20.848Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6ce4ea36-4fa9-4520-b5ac-ebd6a8bcc206","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","name":"stop-kp6pp","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-kp6pp\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"stop-kp6pp"}}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb"}
{"level":"error","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"df501b5d-0207-4e8c-ac62-a03e087abdcb","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-kp6pp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4"}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4"}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:20.857Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"a8082e25-aed7-4c35-9d94-43425c64a1c4","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"3b3da0b1-fdc7-4bc4-90fa-a4e23b5dbc12","rayCluster":"stop-kp6pp","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce"}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce"}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-k92wt"},"reconcileID":"4f941b29-d348-4f7d-a63b-45d455fe6c1d"}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:38:20.899Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"dc499e9c-8721-4a31-9638-c1a5a6cbfdce","name":"stop-kp6pp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:20.903Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"57b3101e-1c64-494b-92e6-f8abefcd0ff5","name":"stop-kp6pp-head"}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192"}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192"}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.911Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.912Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:38:20.912Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","name":"stop-kp6pp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:20.912Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"95374e34-ccb7-40ef-b859-4d23917214cc","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:20.912Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1001b867-87d9-4443-9ac6-2af01503dfe5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:20.912Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1001b867-87d9-4443-9ac6-2af01503dfe5","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:20.912Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1001b867-87d9-4443-9ac6-2af01503dfe5","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:20.912Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"95374e34-ccb7-40ef-b859-4d23917214cc","rayCluster":"stop-kp6pp","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:38:20.912Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1001b867-87d9-4443-9ac6-2af01503dfe5"}
{"level":"info","ts":"2026-06-10T10:38:20.912Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1001b867-87d9-4443-9ac6-2af01503dfe5","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:20.919Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"57b3101e-1c64-494b-92e6-f8abefcd0ff5","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:20.919Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"57b3101e-1c64-494b-92e6-f8abefcd0ff5","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:20.919Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"01f5778f-7a45-4efe-9239-7b6559c50f1b","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:20.919Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","name":"stop-kp6pp","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-kp6pp\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"stop-kp6pp"}}
{"level":"info","ts":"2026-06-10T10:38:20.920Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192"}
{"level":"error","ts":"2026-06-10T10:38:20.920Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e24d42-967c-4a4b-99a0-38463c3fb192","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-kp6pp\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04"}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04"}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:20.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:20.926Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.926Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:20.926Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:20.926Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"05b5a25f-c7d3-44cb-9678-1639d9f66f04","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:21.008Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"01f5778f-7a45-4efe-9239-7b6559c50f1b","name":"stop-kp6pp-head"}
{"level":"info","ts":"2026-06-10T10:38:21.099Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2e22ec04-91ca-4a8b-8dcc-4d6fcb7a4df0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:21.099Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2e22ec04-91ca-4a8b-8dcc-4d6fcb7a4df0","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:21.099Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2e22ec04-91ca-4a8b-8dcc-4d6fcb7a4df0","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:21.099Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2e22ec04-91ca-4a8b-8dcc-4d6fcb7a4df0"}
{"level":"info","ts":"2026-06-10T10:38:21.099Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2e22ec04-91ca-4a8b-8dcc-4d6fcb7a4df0","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:21.103Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"01f5778f-7a45-4efe-9239-7b6559c50f1b","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:21.103Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"01f5778f-7a45-4efe-9239-7b6559c50f1b","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf"}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf"}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:21.277Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"031df66d-a856-442c-9f6b-90721b5ab4cf","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287"}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287"}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:21.294Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"87535810-7801-434b-be23-5505cdf2e287","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e"}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e"}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:21.384Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cced2279-b9d7-4e56-8fe7-956b2a16bc1e","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968"}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968"}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"newHead":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"}}
{"level":"info","ts":"2026-06-10T10:38:21.879Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"331bb2c0-b1e8-4b4d-9d7a-2db4b52f1968","name":"stop-kp6pp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:21.891Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"9d9d70aa-acbb-4a25-b1ce-6678d45156e4","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:21.891Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d598630c-86c6-494d-a9f2-1df17bfb6668","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:21.891Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"9d9d70aa-acbb-4a25-b1ce-6678d45156e4","pod":"stop-kp6pp-head-7s9fj","ip":"10.134.0.32"}
{"level":"info","ts":"2026-06-10T10:38:21.892Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"9d9d70aa-acbb-4a25-b1ce-6678d45156e4","pod":"stop-kp6pp-small-group-worker-5sk92","ip":"10.132.0.49"}
{"level":"info","ts":"2026-06-10T10:38:21.892Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1ea53ddb-cdce-427e-80cc-1df43ecc159f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:21.892Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1ea53ddb-cdce-427e-80cc-1df43ecc159f","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:21.892Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1ea53ddb-cdce-427e-80cc-1df43ecc159f","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:21.892Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1ea53ddb-cdce-427e-80cc-1df43ecc159f"}
{"level":"info","ts":"2026-06-10T10:38:21.892Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"1ea53ddb-cdce-427e-80cc-1df43ecc159f","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:21.903Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d598630c-86c6-494d-a9f2-1df17bfb6668","name":"stop-kp6pp-head"}
{"level":"info","ts":"2026-06-10T10:38:21.903Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"9d9d70aa-acbb-4a25-b1ce-6678d45156e4","certificate":"ray-head-cert-stop-kp6pp","dnsNames":["stop-kp6pp-head-svc","localhost","stop-kp6pp-head-svc.test-ns-k92wt.svc","stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.32","10.132.0.49","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:38:21.913Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d598630c-86c6-494d-a9f2-1df17bfb6668","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:21.913Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d598630c-86c6-494d-a9f2-1df17bfb6668","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:21.914Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"9d9d70aa-acbb-4a25-b1ce-6678d45156e4","certificate":"ray-worker-cert-stop-kp6pp","dnsNames":["stop-kp6pp-worker-svc","localhost","stop-kp6pp-small-group","stop-kp6pp-worker-svc.test-ns-k92wt.svc","stop-kp6pp-worker-svc.test-ns-k92wt.svc.cluster.local","*.stop-kp6pp-worker-svc.test-ns-k92wt.svc","*.stop-kp6pp-worker-svc.test-ns-k92wt.svc.cluster.local","*-worker-*.test-ns-k92wt.svc","*-worker-*.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.32","10.132.0.49","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:38:21.914Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"9d9d70aa-acbb-4a25-b1ce-6678d45156e4","rayCluster":"stop-kp6pp","podIPs":["10.134.0.32","10.132.0.49"]}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea"}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea"}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:23.892Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"68e05cd0-e94d-4055-ae16-7dc583c1deea","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:24.100Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"7f190aa4-41bc-4474-be28-c0c7649b82a0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:24.100Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"7f190aa4-41bc-4474-be28-c0c7649b82a0","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:24.100Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"7f190aa4-41bc-4474-be28-c0c7649b82a0","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:24.100Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"7f190aa4-41bc-4474-be28-c0c7649b82a0"}
{"level":"info","ts":"2026-06-10T10:38:24.100Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"7f190aa4-41bc-4474-be28-c0c7649b82a0","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304"}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304"}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","podName":"stop-kp6pp-head-7s9fj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:26.893Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"30b3bbbf-298d-41ed-823e-56006d0e5304","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:27.101Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"00d0bed3-63c9-45b6-b300-5a2dccc1ac75","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:27.101Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"00d0bed3-63c9-45b6-b300-5a2dccc1ac75","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:27.101Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"00d0bed3-63c9-45b6-b300-5a2dccc1ac75","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:27.101Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"00d0bed3-63c9-45b6-b300-5a2dccc1ac75"}
{"level":"info","ts":"2026-06-10T10:38:27.101Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"00d0bed3-63c9-45b6-b300-5a2dccc1ac75","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf"}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf"}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","podName":"stop-kp6pp-head-7s9fj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","podName":"stop-kp6pp-head-7s9fj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:38:27.901Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"9d4e5313-1156-4323-a047-59106a1480bf","name":"stop-kp6pp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d01ed013-b343-4706-80c2-336101e92a63","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c1c34bf2-e19e-44c7-b08a-ec9c7b860c04","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c1c34bf2-e19e-44c7-b08a-ec9c7b860c04","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c1c34bf2-e19e-44c7-b08a-ec9c7b860c04","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"4d6b2e53-b666-4b59-825d-5941fe51e561","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d01ed013-b343-4706-80c2-336101e92a63","pod":"stop-kp6pp-head-7s9fj","ip":"10.134.0.32"}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d01ed013-b343-4706-80c2-336101e92a63","pod":"stop-kp6pp-small-group-worker-5sk92","ip":"10.132.0.49"}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c1c34bf2-e19e-44c7-b08a-ec9c7b860c04"}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c1c34bf2-e19e-44c7-b08a-ec9c7b860c04","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:27.914Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d01ed013-b343-4706-80c2-336101e92a63","rayCluster":"stop-kp6pp","podIPs":["10.134.0.32","10.132.0.49"]}
{"level":"info","ts":"2026-06-10T10:38:27.927Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"4d6b2e53-b666-4b59-825d-5941fe51e561","name":"stop-kp6pp-head"}
{"level":"info","ts":"2026-06-10T10:38:27.937Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"4d6b2e53-b666-4b59-825d-5941fe51e561","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:27.938Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"4d6b2e53-b666-4b59-825d-5941fe51e561","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552"}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552"}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:29.914Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"b581228e-e1eb-46ae-bedf-8cbf46d2c552","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:30.101Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0ea85f3a-8968-489d-8109-298032b87dae","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:30.101Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0ea85f3a-8968-489d-8109-298032b87dae","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:30.102Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0ea85f3a-8968-489d-8109-298032b87dae","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:30.102Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0ea85f3a-8968-489d-8109-298032b87dae"}
{"level":"info","ts":"2026-06-10T10:38:30.102Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0ea85f3a-8968-489d-8109-298032b87dae","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4"}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4"}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","podName":"stop-kp6pp-head-7s9fj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","podName":"stop-kp6pp-head-7s9fj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:31.439Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"2c35a51c-de46-476c-8b58-0a9c85692cf4","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c"}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c"}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","podName":"stop-kp6pp-head-7s9fj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","podName":"stop-kp6pp-head-7s9fj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:38:32.442Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":1,"oldDesiredWorkerReplicas":1,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":1,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":1,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:38:32.443Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"73446854-6ed4-4214-b45d-aea8854bd63c","name":"stop-kp6pp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"2d8a303a-57a6-4c2e-8756-fd4c4df8ece2","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"3bbbef6f-dc38-4a00-afd0-532601cdb020","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"2d8a303a-57a6-4c2e-8756-fd4c4df8ece2","pod":"stop-kp6pp-head-7s9fj","ip":"10.134.0.32"}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"2d8a303a-57a6-4c2e-8756-fd4c4df8ece2","pod":"stop-kp6pp-small-group-worker-5sk92","ip":"10.132.0.49"}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"2d8a303a-57a6-4c2e-8756-fd4c4df8ece2","rayCluster":"stop-kp6pp","podIPs":["10.134.0.32","10.132.0.49"]}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"70607981-8fab-4a2e-a428-72c68955dd49","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"70607981-8fab-4a2e-a428-72c68955dd49","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"70607981-8fab-4a2e-a428-72c68955dd49","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"70607981-8fab-4a2e-a428-72c68955dd49"}
{"level":"info","ts":"2026-06-10T10:38:32.454Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"70607981-8fab-4a2e-a428-72c68955dd49","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:32.464Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"3bbbef6f-dc38-4a00-afd0-532601cdb020","name":"stop-kp6pp-head"}
{"level":"info","ts":"2026-06-10T10:38:32.473Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"3bbbef6f-dc38-4a00-afd0-532601cdb020","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:32.473Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"3bbbef6f-dc38-4a00-afd0-532601cdb020","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:33.102Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2696126e-eea0-4c69-a52b-feaa06da139e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:33.102Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2696126e-eea0-4c69-a52b-feaa06da139e","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:33.102Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2696126e-eea0-4c69-a52b-feaa06da139e","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:33.102Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2696126e-eea0-4c69-a52b-feaa06da139e"}
{"level":"info","ts":"2026-06-10T10:38:33.102Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2696126e-eea0-4c69-a52b-feaa06da139e","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e"}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e"}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","podName":"stop-kp6pp-head-7s9fj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","podName":"stop-kp6pp-head-7s9fj","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:34.454Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"cc1f10ee-2994-45f2-a1b8-92cdf4998c7e","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:36.103Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52e2f7f3-f990-4c0b-be33-452ca7beb47e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:36.103Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52e2f7f3-f990-4c0b-be33-452ca7beb47e","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:36.103Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52e2f7f3-f990-4c0b-be33-452ca7beb47e","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:36.103Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52e2f7f3-f990-4c0b-be33-452ca7beb47e"}
{"level":"info","ts":"2026-06-10T10:38:36.103Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52e2f7f3-f990-4c0b-be33-452ca7beb47e","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5"}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5"}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:38:38.934Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"ed6cd2cc-952e-4eba-bf3a-5b1874caf2c5","name":"stop-kp6pp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"06eb18b5-e0c6-4c3b-830f-2b18c7f2e1f1","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"055ca2b5-aff0-4b45-b137-229f55a7bd61","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"06eb18b5-e0c6-4c3b-830f-2b18c7f2e1f1","pod":"stop-kp6pp-head-7s9fj","ip":"10.134.0.32"}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"06eb18b5-e0c6-4c3b-830f-2b18c7f2e1f1","pod":"stop-kp6pp-small-group-worker-5sk92","ip":"10.132.0.49"}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"02b31d0e-1a2c-4292-8908-a9deb1031296","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"02b31d0e-1a2c-4292-8908-a9deb1031296","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"02b31d0e-1a2c-4292-8908-a9deb1031296","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"06eb18b5-e0c6-4c3b-830f-2b18c7f2e1f1","rayCluster":"stop-kp6pp","podIPs":["10.134.0.32","10.132.0.49"]}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"02b31d0e-1a2c-4292-8908-a9deb1031296"}
{"level":"info","ts":"2026-06-10T10:38:38.945Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"02b31d0e-1a2c-4292-8908-a9deb1031296","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:38.954Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"055ca2b5-aff0-4b45-b137-229f55a7bd61","name":"stop-kp6pp-head"}
{"level":"info","ts":"2026-06-10T10:38:38.961Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"055ca2b5-aff0-4b45-b137-229f55a7bd61","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:38.961Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"055ca2b5-aff0-4b45-b137-229f55a7bd61","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:39.104Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9ea738d-6abf-49bf-98c5-c693a0c4fbaa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:39.104Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9ea738d-6abf-49bf-98c5-c693a0c4fbaa","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:39.104Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9ea738d-6abf-49bf-98c5-c693a0c4fbaa","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:39.104Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9ea738d-6abf-49bf-98c5-c693a0c4fbaa"}
{"level":"info","ts":"2026-06-10T10:38:39.104Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9ea738d-6abf-49bf-98c5-c693a0c4fbaa","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d"}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d"}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","podName":"stop-kp6pp-small-group-worker-5sk92","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:38:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:40.946Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"f8508a7b-43a6-4c61-9b57-48c5cbae0f3d","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:42.106Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f642eb0f-0987-42bb-b16d-df3c2ea7de1c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:42.106Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f642eb0f-0987-42bb-b16d-df3c2ea7de1c","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:42.106Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f642eb0f-0987-42bb-b16d-df3c2ea7de1c","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:42.106Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f642eb0f-0987-42bb-b16d-df3c2ea7de1c"}
{"level":"info","ts":"2026-06-10T10:38:42.106Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f642eb0f-0987-42bb-b16d-df3c2ea7de1c","RayCluster":"stop-kp6pp","State":""}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162"}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162"}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","oldState":"","newState":"ready","oldReason":"","newReason":""}
{"level":"info","ts":"2026-06-10T10:38:43.468Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"c2fe9f31-f2ea-4a73-9b98-c364d4234162","name":"stop-kp6pp","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d243424f-697b-4b39-a0b9-ac424bf67fdb","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1d4f64ab-3b2c-423e-8ae9-bee17faa14c1","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d243424f-697b-4b39-a0b9-ac424bf67fdb","pod":"stop-kp6pp-head-7s9fj","ip":"10.134.0.32"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d243424f-697b-4b39-a0b9-ac424bf67fdb","pod":"stop-kp6pp-small-group-worker-5sk92","ip":"10.132.0.49"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"d243424f-697b-4b39-a0b9-ac424bf67fdb","rayCluster":"stop-kp6pp","podIPs":["10.134.0.32","10.132.0.49"]}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","head service name":"stop-kp6pp-head-svc","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","head service URL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3"}
{"level":"info","ts":"2026-06-10T10:38:43.481Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","command":["if","!","ray","job","status","--address","http://stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","stop-4cc45",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","--no-wait","--submission-id","stop-4cc45","--","python /home/ray/jobs/stop.py",";","fi",";","ray","job","logs","--address","http://stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","--follow","stop-4cc45"]}
{"level":"info","ts":"2026-06-10T10:38:43.490Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-06-10T10:38:43.490Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","SubmissionMode":"K8sJobMode","RayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:38:43.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"fa7713dd-b498-45f5-b18c-6e0e2f18f2a3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:38:43.495Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1d4f64ab-3b2c-423e-8ae9-bee17faa14c1","name":"stop-kp6pp-head"}
{"level":"info","ts":"2026-06-10T10:38:43.501Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:43.501Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.501Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:43.502Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5"}
{"level":"info","ts":"2026-06-10T10:38:43.502Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","head service name":"stop-kp6pp-head-svc","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:38:43.502Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","head service URL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:38:43.502Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-06-10T10:38:43.502Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","SubmissionMode":"K8sJobMode","RayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.502Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:38:43.502Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:38:43.509Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-10T10:38:43.509Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5"}
{"level":"error","ts":"2026-06-10T10:38:43.509Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"b91900e5-8fe1-45b1-8a2e-89bbaf2025d5","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.509Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"cee651f5-b6dd-4663-8c1f-2e338860dda1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:43.509Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"cee651f5-b6dd-4663-8c1f-2e338860dda1","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.509Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"cee651f5-b6dd-4663-8c1f-2e338860dda1","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:43.509Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"cee651f5-b6dd-4663-8c1f-2e338860dda1"}
{"level":"info","ts":"2026-06-10T10:38:43.512Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1d4f64ab-3b2c-423e-8ae9-bee17faa14c1","name":"stop-kp6pp-workers"}
{"level":"info","ts":"2026-06-10T10:38:43.512Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1d4f64ab-3b2c-423e-8ae9-bee17faa14c1","cluster":"stop-kp6pp"}
{"level":"error","ts":"2026-06-10T10:38:43.521Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"cee651f5-b6dd-4663-8c1f-2e338860dda1","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.522Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"cee651f5-b6dd-4663-8c1f-2e338860dda1"}
{"level":"error","ts":"2026-06-10T10:38:43.522Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"cee651f5-b6dd-4663-8c1f-2e338860dda1","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.522Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2d336af1-6bbf-4b45-9b56-9110181f00de","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:43.522Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2d336af1-6bbf-4b45-9b56-9110181f00de","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.522Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2d336af1-6bbf-4b45-9b56-9110181f00de","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:43.522Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2d336af1-6bbf-4b45-9b56-9110181f00de"}
{"level":"error","ts":"2026-06-10T10:38:43.527Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2d336af1-6bbf-4b45-9b56-9110181f00de","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.527Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2d336af1-6bbf-4b45-9b56-9110181f00de"}
{"level":"error","ts":"2026-06-10T10:38:43.527Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2d336af1-6bbf-4b45-9b56-9110181f00de","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0f7755dc-8e7b-40c5-b12f-a922d152874d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:43.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0f7755dc-8e7b-40c5-b12f-a922d152874d","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0f7755dc-8e7b-40c5-b12f-a922d152874d","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:43.599Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0f7755dc-8e7b-40c5-b12f-a922d152874d"}
{"level":"error","ts":"2026-06-10T10:38:43.604Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0f7755dc-8e7b-40c5-b12f-a922d152874d","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.604Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0f7755dc-8e7b-40c5-b12f-a922d152874d"}
{"level":"error","ts":"2026-06-10T10:38:43.604Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0f7755dc-8e7b-40c5-b12f-a922d152874d","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.645Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0e38e738-8ed0-4ce1-b195-31663cd4b449","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:43.645Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0e38e738-8ed0-4ce1-b195-31663cd4b449","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.645Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0e38e738-8ed0-4ce1-b195-31663cd4b449","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:43.645Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0e38e738-8ed0-4ce1-b195-31663cd4b449"}
{"level":"error","ts":"2026-06-10T10:38:43.650Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0e38e738-8ed0-4ce1-b195-31663cd4b449","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.650Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0e38e738-8ed0-4ce1-b195-31663cd4b449"}
{"level":"error","ts":"2026-06-10T10:38:43.650Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"0e38e738-8ed0-4ce1-b195-31663cd4b449","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.730Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52ff3ba6-359e-491e-8a44-d493fabf2b6c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:43.731Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52ff3ba6-359e-491e-8a44-d493fabf2b6c","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52ff3ba6-359e-491e-8a44-d493fabf2b6c","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:43.731Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52ff3ba6-359e-491e-8a44-d493fabf2b6c"}
{"level":"error","ts":"2026-06-10T10:38:43.736Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52ff3ba6-359e-491e-8a44-d493fabf2b6c","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.736Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52ff3ba6-359e-491e-8a44-d493fabf2b6c"}
{"level":"error","ts":"2026-06-10T10:38:43.736Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"52ff3ba6-359e-491e-8a44-d493fabf2b6c","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.897Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d3d6b4de-ab52-4587-bcf1-36e6e6fe289b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:43.897Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d3d6b4de-ab52-4587-bcf1-36e6e6fe289b","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:43.897Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d3d6b4de-ab52-4587-bcf1-36e6e6fe289b","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:43.897Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d3d6b4de-ab52-4587-bcf1-36e6e6fe289b"}
{"level":"error","ts":"2026-06-10T10:38:43.908Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d3d6b4de-ab52-4587-bcf1-36e6e6fe289b","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:43.908Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d3d6b4de-ab52-4587-bcf1-36e6e6fe289b"}
{"level":"error","ts":"2026-06-10T10:38:43.908Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d3d6b4de-ab52-4587-bcf1-36e6e6fe289b","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:44.229Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5d46f984-bffe-4d2b-a510-93cba0ae9340","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:44.229Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5d46f984-bffe-4d2b-a510-93cba0ae9340","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:44.229Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5d46f984-bffe-4d2b-a510-93cba0ae9340","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:44.229Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5d46f984-bffe-4d2b-a510-93cba0ae9340"}
{"level":"error","ts":"2026-06-10T10:38:44.235Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5d46f984-bffe-4d2b-a510-93cba0ae9340","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:44.235Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5d46f984-bffe-4d2b-a510-93cba0ae9340"}
{"level":"error","ts":"2026-06-10T10:38:44.235Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5d46f984-bffe-4d2b-a510-93cba0ae9340","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:44.876Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2cc5cb86-834c-47b8-b492-4e7e06016aeb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:44.876Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2cc5cb86-834c-47b8-b492-4e7e06016aeb","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:44.876Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2cc5cb86-834c-47b8-b492-4e7e06016aeb","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:44.876Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2cc5cb86-834c-47b8-b492-4e7e06016aeb"}
{"level":"error","ts":"2026-06-10T10:38:44.882Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2cc5cb86-834c-47b8-b492-4e7e06016aeb","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:44.882Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2cc5cb86-834c-47b8-b492-4e7e06016aeb"}
{"level":"error","ts":"2026-06-10T10:38:44.882Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"2cc5cb86-834c-47b8-b492-4e7e06016aeb","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28"}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28"}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","1 head service found":"stop-kp6pp-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","Found 1 head Pod":"stop-kp6pp-head-7s9fj","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","head Pod":"stop-kp6pp-head-7s9fj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-kp6pp-head-7s9fj. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","worker Pod":"stop-kp6pp-small-group-worker-5sk92","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-kp6pp-small-group-worker-5sk92. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:45.481Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"04fa6d7d-9d98-4816-9d5d-4968cf2d0d28","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:45.978Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d0c4bf80-93b9-471f-9d6f-209fd2847df4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:45.978Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d0c4bf80-93b9-471f-9d6f-209fd2847df4","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:45.978Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d0c4bf80-93b9-471f-9d6f-209fd2847df4","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:45.978Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d0c4bf80-93b9-471f-9d6f-209fd2847df4"}
{"level":"error","ts":"2026-06-10T10:38:45.984Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d0c4bf80-93b9-471f-9d6f-209fd2847df4","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:45.984Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d0c4bf80-93b9-471f-9d6f-209fd2847df4"}
{"level":"error","ts":"2026-06-10T10:38:45.984Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"d0c4bf80-93b9-471f-9d6f-209fd2847df4","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:46.163Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9304689-65d7-433d-9c8a-e42bc5ab56cf","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:46.163Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9304689-65d7-433d-9c8a-e42bc5ab56cf","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:46.163Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9304689-65d7-433d-9c8a-e42bc5ab56cf","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:46.163Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9304689-65d7-433d-9c8a-e42bc5ab56cf"}
{"level":"error","ts":"2026-06-10T10:38:46.168Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9304689-65d7-433d-9c8a-e42bc5ab56cf","JobId":"stop-4cc45","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:46.168Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9304689-65d7-433d-9c8a-e42bc5ab56cf"}
{"level":"error","ts":"2026-06-10T10:38:46.168Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"c9304689-65d7-433d-9c8a-e42bc5ab56cf","error":"Job stop-4cc45 does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:38:46.915Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d0beb9a3-b559-4d20-a3a4-452729150957","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:38:46.915Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d0beb9a3-b559-4d20-a3a4-452729150957","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:38:46.915Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d0beb9a3-b559-4d20-a3a4-452729150957","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:38:46.915Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"d0beb9a3-b559-4d20-a3a4-452729150957","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:38:48.320Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"5cde167b-f754-40b6-9639-e07fbaa7c359","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:48.320Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"5cde167b-f754-40b6-9639-e07fbaa7c359","pod":"stop-kp6pp-head-7s9fj","ip":"10.134.0.32"}
{"level":"info","ts":"2026-06-10T10:38:48.320Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"5cde167b-f754-40b6-9639-e07fbaa7c359","pod":"stop-kp6pp-small-group-worker-5sk92","ip":"10.132.0.49"}
{"level":"info","ts":"2026-06-10T10:38:48.320Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"5cde167b-f754-40b6-9639-e07fbaa7c359","rayCluster":"stop-kp6pp","podIPs":["10.134.0.32","10.132.0.49"]}
{"level":"info","ts":"2026-06-10T10:38:51.289Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"9a43d177-b13b-4b62-ae90-80bb7ae3f422","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:51.289Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"9a43d177-b13b-4b62-ae90-80bb7ae3f422","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:51.289Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"9a43d177-b13b-4b62-ae90-80bb7ae3f422","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:51.290Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"9a43d177-b13b-4b62-ae90-80bb7ae3f422"}
{"level":"info","ts":"2026-06-10T10:38:51.296Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"9a43d177-b13b-4b62-ae90-80bb7ae3f422","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:38:51.296Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"9a43d177-b13b-4b62-ae90-80bb7ae3f422","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:38:51.309Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e6d0e309-7e74-4970-9a33-88c2ab6431c3","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:51.309Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e6d0e309-7e74-4970-9a33-88c2ab6431c3","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:51.309Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e6d0e309-7e74-4970-9a33-88c2ab6431c3","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:51.309Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e6d0e309-7e74-4970-9a33-88c2ab6431c3"}
{"level":"info","ts":"2026-06-10T10:38:51.312Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e6d0e309-7e74-4970-9a33-88c2ab6431c3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e"}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e"}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","Found 1 head Pod":"raycluster-head-jjvtb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","head Pod":"raycluster-head-jjvtb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-jjvtb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","worker Pod":"raycluster-small-group-worker-7j6bp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-7j6bp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:38:53.282Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"reconcileID":"930566a9-3f14-4e93-bf71-06caa72be26e","seconds":300}
{"level":"info","ts":"2026-06-10T10:38:54.310Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8df53852-9a02-41ed-b571-2090571c95a1","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:54.310Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8df53852-9a02-41ed-b571-2090571c95a1","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:54.310Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8df53852-9a02-41ed-b571-2090571c95a1","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:54.310Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8df53852-9a02-41ed-b571-2090571c95a1"}
{"level":"info","ts":"2026-06-10T10:38:54.314Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8df53852-9a02-41ed-b571-2090571c95a1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:38:54.314Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8df53852-9a02-41ed-b571-2090571c95a1","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:38:54.327Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e7c7dffc-7295-4c98-bd26-86f301a2f731","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:54.327Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e7c7dffc-7295-4c98-bd26-86f301a2f731","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:54.327Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e7c7dffc-7295-4c98-bd26-86f301a2f731","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:54.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e7c7dffc-7295-4c98-bd26-86f301a2f731"}
{"level":"info","ts":"2026-06-10T10:38:54.330Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"e7c7dffc-7295-4c98-bd26-86f301a2f731","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:38:57.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5dc4ca77-2c10-4b06-a8b5-c740deeb7090","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:38:57.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5dc4ca77-2c10-4b06-a8b5-c740deeb7090","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:38:57.327Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5dc4ca77-2c10-4b06-a8b5-c740deeb7090","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:38:57.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5dc4ca77-2c10-4b06-a8b5-c740deeb7090"}
{"level":"info","ts":"2026-06-10T10:38:57.330Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5dc4ca77-2c10-4b06-a8b5-c740deeb7090","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:00.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6a2567bc-dd49-40bf-a644-417b5262f6ca","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:00.331Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6a2567bc-dd49-40bf-a644-417b5262f6ca","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:00.331Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6a2567bc-dd49-40bf-a644-417b5262f6ca","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:00.331Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6a2567bc-dd49-40bf-a644-417b5262f6ca"}
{"level":"info","ts":"2026-06-10T10:39:00.334Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"6a2567bc-dd49-40bf-a644-417b5262f6ca","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:03.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a5b7e112-1c56-4f42-8802-ebe9bd3e4ee3","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:03.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a5b7e112-1c56-4f42-8802-ebe9bd3e4ee3","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:03.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a5b7e112-1c56-4f42-8802-ebe9bd3e4ee3","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:03.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a5b7e112-1c56-4f42-8802-ebe9bd3e4ee3"}
{"level":"info","ts":"2026-06-10T10:39:03.338Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a5b7e112-1c56-4f42-8802-ebe9bd3e4ee3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:06.339Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4cdcdc28-f3da-472f-b8f4-c52b85372bbf","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:06.339Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4cdcdc28-f3da-472f-b8f4-c52b85372bbf","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:06.339Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4cdcdc28-f3da-472f-b8f4-c52b85372bbf","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:06.339Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4cdcdc28-f3da-472f-b8f4-c52b85372bbf"}
{"level":"info","ts":"2026-06-10T10:39:06.342Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4cdcdc28-f3da-472f-b8f4-c52b85372bbf","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:09.343Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"60e1f17d-30e3-4881-81d3-e20701d77be8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:09.343Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"60e1f17d-30e3-4881-81d3-e20701d77be8","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:09.343Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"60e1f17d-30e3-4881-81d3-e20701d77be8","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:09.343Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"60e1f17d-30e3-4881-81d3-e20701d77be8"}
{"level":"info","ts":"2026-06-10T10:39:09.346Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"60e1f17d-30e3-4881-81d3-e20701d77be8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:12.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a0ff91c5-7c32-4ea3-a42d-36f8fecded9f","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:12.347Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a0ff91c5-7c32-4ea3-a42d-36f8fecded9f","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:12.347Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a0ff91c5-7c32-4ea3-a42d-36f8fecded9f","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:12.347Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a0ff91c5-7c32-4ea3-a42d-36f8fecded9f"}
{"level":"info","ts":"2026-06-10T10:39:12.350Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"a0ff91c5-7c32-4ea3-a42d-36f8fecded9f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:15.351Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"375c1e70-a8fa-44f3-8260-6102f5dca11a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:15.351Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"375c1e70-a8fa-44f3-8260-6102f5dca11a","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:15.351Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"375c1e70-a8fa-44f3-8260-6102f5dca11a","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:15.351Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"375c1e70-a8fa-44f3-8260-6102f5dca11a"}
{"level":"info","ts":"2026-06-10T10:39:15.354Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"375c1e70-a8fa-44f3-8260-6102f5dca11a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:15.354Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"375c1e70-a8fa-44f3-8260-6102f5dca11a","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:39:15.371Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8cc1aeed-09d9-40c8-84ed-6a6085f09b5d","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:15.371Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8cc1aeed-09d9-40c8-84ed-6a6085f09b5d","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:15.371Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8cc1aeed-09d9-40c8-84ed-6a6085f09b5d","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:15.371Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8cc1aeed-09d9-40c8-84ed-6a6085f09b5d"}
{"level":"info","ts":"2026-06-10T10:39:15.374Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"8cc1aeed-09d9-40c8-84ed-6a6085f09b5d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:18.372Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4aab7d30-ae4d-4287-9aa2-1bdbffffa4da","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:18.372Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4aab7d30-ae4d-4287-9aa2-1bdbffffa4da","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:18.372Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4aab7d30-ae4d-4287-9aa2-1bdbffffa4da","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:18.372Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4aab7d30-ae4d-4287-9aa2-1bdbffffa4da"}
{"level":"info","ts":"2026-06-10T10:39:18.375Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4aab7d30-ae4d-4287-9aa2-1bdbffffa4da","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:19.114Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"009c42d4-0d0a-4d0b-8db3-ceb7c8868956","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:19.114Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"009c42d4-0d0a-4d0b-8db3-ceb7c8868956","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:19.114Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"009c42d4-0d0a-4d0b-8db3-ceb7c8868956","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:19.114Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"009c42d4-0d0a-4d0b-8db3-ceb7c8868956"}
{"level":"info","ts":"2026-06-10T10:39:19.118Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"009c42d4-0d0a-4d0b-8db3-ceb7c8868956","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:20.123Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"94bde56f-11be-426f-8a6e-928c2b3c8b34","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:20.123Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"94bde56f-11be-426f-8a6e-928c2b3c8b34","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.123Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"94bde56f-11be-426f-8a6e-928c2b3c8b34","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:20.123Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"94bde56f-11be-426f-8a6e-928c2b3c8b34"}
{"level":"info","ts":"2026-06-10T10:39:20.126Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"94bde56f-11be-426f-8a6e-928c2b3c8b34","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:20.140Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f97dd0f4-6c25-4776-be3a-6e8576e54139","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:20.140Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f97dd0f4-6c25-4776-be3a-6e8576e54139","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.140Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f97dd0f4-6c25-4776-be3a-6e8576e54139","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:20.140Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f97dd0f4-6c25-4776-be3a-6e8576e54139"}
{"level":"info","ts":"2026-06-10T10:39:20.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f97dd0f4-6c25-4776-be3a-6e8576e54139","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:38:51Z","endTime":"2026-06-10T10:39:13Z"},"jobId":"stop-4cc45","rayClusterName":"stop-kp6pp","dashboardURL":"stop-kp6pp-head-svc.test-ns-k92wt.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-06-10T10:38:18Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:38:43Z","stateTransitionTimes":{"ready":"2026-06-10T10:38:43Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.32","serviceIP":"10.134.0.32","podName":"stop-kp6pp-head-7s9fj","serviceName":"stop-kp6pp-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:38:38Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:38:43Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:38:18Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:38:18Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:20.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"f97dd0f4-6c25-4776-be3a-6e8576e54139","old JobStatus":"STOPPED","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-10T10:39:20.156Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4bf33bad-e9ec-493a-ae8b-80cafeea06e5","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:20.156Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4bf33bad-e9ec-493a-ae8b-80cafeea06e5","ShutdownAfterJobFinishes":false,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:39:20 +0000 UTC","Now":"2026-06-10T10:39:20.156Z","ShutdownTime":"2026-06-10T10:39:20.000Z"}
{"level":"info","ts":"2026-06-10T10:39:20.431Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"781f4344-5d2e-4bf5-90c8-92706feb5cc8","DeletionTimestamp":"2026-06-10 10:39:20 +0000 UTC"}
{"level":"info","ts":"2026-06-10T10:39:20.431Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"781f4344-5d2e-4bf5-90c8-92706feb5cc8","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-06-10T10:39:20.445Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"131ed178-2f95-47b0-b149-8cf02bb32f60","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.445Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"131ed178-2f95-47b0-b149-8cf02bb32f60"}
{"level":"error","ts":"2026-06-10T10:39:20.445Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"131ed178-2f95-47b0-b149-8cf02bb32f60","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.446Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"5ac6efde-7a69-4ca4-a920-f34a6be39ef7"}
{"level":"error","ts":"2026-06-10T10:39:20.451Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"19d1162f-9a51-4b3c-92af-b6e17ca8bad1","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.451Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"19d1162f-9a51-4b3c-92af-b6e17ca8bad1"}
{"level":"error","ts":"2026-06-10T10:39:20.451Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"19d1162f-9a51-4b3c-92af-b6e17ca8bad1","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.460Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"4d4fab8b-d022-448e-8150-af387fdf8bb8"}
{"level":"error","ts":"2026-06-10T10:39:20.462Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"7c0d377e-7a99-47fd-9fb2-7c662ab0166e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.462Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"7c0d377e-7a99-47fd-9fb2-7c662ab0166e"}
{"level":"error","ts":"2026-06-10T10:39:20.462Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"7c0d377e-7a99-47fd-9fb2-7c662ab0166e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1fb22e5d-2f56-43dc-9a52-5a0273b90473","namespacedName":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1fb22e5d-2f56-43dc-9a52-5a0273b90473","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1fb22e5d-2f56-43dc-9a52-5a0273b90473","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"36283a34-3031-4a15-8969-0a94f243b350","rayCluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"reconcileID":"fee104b7-2a0b-4672-b3c6-cf62b2388d48"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"7020cb07-f894-4506-820c-d3f916a23dd8"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1fb22e5d-2f56-43dc-9a52-5a0273b90473","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"22ef9071-2cca-47fa-aac9-988a7a061a18"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"36283a34-3031-4a15-8969-0a94f243b350","pod":"stop-kp6pp-head-7s9fj","ip":"10.134.0.32"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"36283a34-3031-4a15-8969-0a94f243b350","pod":"stop-kp6pp-small-group-worker-5sk92","ip":"10.132.0.49"}
{"level":"info","ts":"2026-06-10T10:39:20.465Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"36283a34-3031-4a15-8969-0a94f243b350","rayCluster":"stop-kp6pp","podIPs":["10.134.0.32","10.132.0.49"]}
{"level":"info","ts":"2026-06-10T10:39:20.471Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1fb22e5d-2f56-43dc-9a52-5a0273b90473","name":"kuberay-gateway-access","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:20.477Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1fb22e5d-2f56-43dc-9a52-5a0273b90473","name":"test-ns-k92wt-stop-kp6pp","namespace":"opendatahub"}
{"level":"info","ts":"2026-06-10T10:39:20.481Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1fb22e5d-2f56-43dc-9a52-5a0273b90473","configMap":"kube-rbac-proxy-config-stop-kp6pp"}
{"level":"error","ts":"2026-06-10T10:39:20.483Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"cdb0c59b-dd19-4df1-9b16-24ed5b0d6ece","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.483Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"cdb0c59b-dd19-4df1-9b16-24ed5b0d6ece"}
{"level":"error","ts":"2026-06-10T10:39:20.483Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"cdb0c59b-dd19-4df1-9b16-24ed5b0d6ece","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.486Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1fb22e5d-2f56-43dc-9a52-5a0273b90473","serviceAccount":"stop-kp6pp-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:39:20.493Z","logger":"raycluster-resource","msg":"default","name":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.493Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"stop-kp6pp","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:20.502Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"1fb22e5d-2f56-43dc-9a52-5a0273b90473","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.502Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"07114ed5-32a0-4f9a-b9b4-9c15bdd565b4","namespacedName":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:20.502Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"07114ed5-32a0-4f9a-b9b4-9c15bdd565b4","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.502Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"07114ed5-32a0-4f9a-b9b4-9c15bdd565b4","cluster":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.502Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"07114ed5-32a0-4f9a-b9b4-9c15bdd565b4","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:39:20.502Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"07114ed5-32a0-4f9a-b9b4-9c15bdd565b4","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:39:20.502Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"e66eebc1-4d66-46ff-b3c9-e620b33b970c"}
{"level":"info","ts":"2026-06-10T10:39:20.502Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"64e7daa6-9337-4e4a-af36-faf89301bbe3"}
{"level":"info","ts":"2026-06-10T10:39:20.502Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"64d88e78-5dad-4667-bccf-2de6d1c342da","namespace":"test-ns-k92wt","clusterName":"stop-kp6pp"}
{"level":"error","ts":"2026-06-10T10:39:20.505Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"07114ed5-32a0-4f9a-b9b4-9c15bdd565b4","error":"rayclusters.ray.io \"stop-kp6pp\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.506Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"07114ed5-32a0-4f9a-b9b4-9c15bdd565b4"}
{"level":"error","ts":"2026-06-10T10:39:20.506Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"07114ed5-32a0-4f9a-b9b4-9c15bdd565b4","error":"rayclusters.ray.io \"stop-kp6pp\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.506Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"056e06e8-bb28-4798-acac-caf9569f31f7","namespacedName":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:20.506Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"056e06e8-bb28-4798-acac-caf9569f31f7"}
{"level":"info","ts":"2026-06-10T10:39:20.506Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"056e06e8-bb28-4798-acac-caf9569f31f7","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:39:20.506Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"056e06e8-bb28-4798-acac-caf9569f31f7","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:20.508Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c396312a-a509-42bd-ad18-807ae0940a87"}
{"level":"info","ts":"2026-06-10T10:39:20.510Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"38c3c209-dbee-4fc9-b1be-96c2d865bbce","namespacedName":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:20.510Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"38c3c209-dbee-4fc9-b1be-96c2d865bbce"}
{"level":"info","ts":"2026-06-10T10:39:20.510Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"38c3c209-dbee-4fc9-b1be-96c2d865bbce","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:39:20.510Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"38c3c209-dbee-4fc9-b1be-96c2d865bbce","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:20.511Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c7e029f4-ac55-4ebe-83cd-a00d4bf3ce95","namespacedName":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:20.511Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c7e029f4-ac55-4ebe-83cd-a00d4bf3ce95"}
{"level":"info","ts":"2026-06-10T10:39:20.511Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c7e029f4-ac55-4ebe-83cd-a00d4bf3ce95","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:39:20.511Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"c7e029f4-ac55-4ebe-83cd-a00d4bf3ce95","namespace":"test-ns-k92wt"}
{"level":"error","ts":"2026-06-10T10:39:20.512Z","msg":"Failed to delete SelfSigned issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"64d88e78-5dad-4667-bccf-2de6d1c342da","issuer":"ray-selfsigned-issuer-stop-kp6pp","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-stop-kp6pp\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:254\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:39:20.519Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"64d88e78-5dad-4667-bccf-2de6d1c342da","certificate":"ray-worker-cert-stop-kp6pp","error":"certificates.cert-manager.io \"ray-worker-cert-stop-kp6pp\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.521Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"a8adb36b-8f7f-4c9f-a775-7818488165ee"}
{"level":"error","ts":"2026-06-10T10:39:20.523Z","msg":"Failed to delete CA certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"64d88e78-5dad-4667-bccf-2de6d1c342da","certificate":"ray-ca-certificate-stop-kp6pp","error":"certificates.cert-manager.io \"ray-ca-certificate-stop-kp6pp\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:275\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:39:20.523Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"259fa458-1add-4141-a8b6-b43065eac759","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.523Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"259fa458-1add-4141-a8b6-b43065eac759"}
{"level":"error","ts":"2026-06-10T10:39:20.523Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"259fa458-1add-4141-a8b6-b43065eac759","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.599Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"64d88e78-5dad-4667-bccf-2de6d1c342da","errorCount":3}
{"level":"info","ts":"2026-06-10T10:39:20.599Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"64d88e78-5dad-4667-bccf-2de6d1c342da"}
{"level":"error","ts":"2026-06-10T10:39:20.599Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"64d88e78-5dad-4667-bccf-2de6d1c342da","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-stop-kp6pp\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:39:20.604Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"a2db9d42-8b05-4ee5-b499-f117b11ff427","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.604Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"a2db9d42-8b05-4ee5-b499-f117b11ff427"}
{"level":"error","ts":"2026-06-10T10:39:20.604Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"a2db9d42-8b05-4ee5-b499-f117b11ff427","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.604Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"8a206ccb-402c-418e-b29d-644a695d0b04","namespace":"test-ns-k92wt","clusterName":"stop-kp6pp"}
{"level":"info","ts":"2026-06-10T10:39:20.604Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-kp6pp","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"stop-kp6pp","reconcileID":"8a206ccb-402c-418e-b29d-644a695d0b04","namespace":"test-ns-k92wt","clusterName":"stop-kp6pp"}
{"level":"error","ts":"2026-06-10T10:39:20.764Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"b0159f0b-450a-4e81-88d8-567c6fb5f57e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:20.765Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"b0159f0b-450a-4e81-88d8-567c6fb5f57e"}
{"level":"error","ts":"2026-06-10T10:39:20.765Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"b0159f0b-450a-4e81-88d8-567c6fb5f57e","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:39:21.085Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"879eca7b-321b-4b62-b1f4-928c88b7c6c0","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:21.085Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"879eca7b-321b-4b62-b1f4-928c88b7c6c0"}
{"level":"error","ts":"2026-06-10T10:39:21.085Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"879eca7b-321b-4b62-b1f4-928c88b7c6c0","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:21.375Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"stop","namespace":"test-ns-k92wt"},"reconcileID":"66fc1197-2888-4fd6-b0a5-ea29d7e61b27"}
{"level":"error","ts":"2026-06-10T10:39:21.726Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"7d335215-aafe-49a0-a020-16f2ce8f876b","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:21.726Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"7d335215-aafe-49a0-a020-16f2ce8f876b"}
{"level":"error","ts":"2026-06-10T10:39:21.726Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"7d335215-aafe-49a0-a020-16f2ce8f876b","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:39:23.006Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"97659240-77b0-4239-ae7f-0721e275ccbe","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:23.006Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"97659240-77b0-4239-ae7f-0721e275ccbe"}
{"level":"error","ts":"2026-06-10T10:39:23.006Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"97659240-77b0-4239-ae7f-0721e275ccbe","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.456Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"d9bad60d-7680-43ac-8146-5506d45d3019","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:25.456Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"d9bad60d-7680-43ac-8146-5506d45d3019","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:39:25.468Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"d9bad60d-7680-43ac-8146-5506d45d3019"}
{"level":"info","ts":"2026-06-10T10:39:25.468Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"d9bad60d-7680-43ac-8146-5506d45d3019","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:39:25.468Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"d9bad60d-7680-43ac-8146-5506d45d3019","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-pvbmz","rayClusterName":"long-running-v8ndd","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:39:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:39:25.468Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"d9bad60d-7680-43ac-8146-5506d45d3019","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:39:25.480Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"46fa5a5c-5890-475e-885f-b82c14ed6eb0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:25.480Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"46fa5a5c-5890-475e-885f-b82c14ed6eb0","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.480Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"46fa5a5c-5890-475e-885f-b82c14ed6eb0","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.480Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"46fa5a5c-5890-475e-885f-b82c14ed6eb0","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.485Z","logger":"raycluster-resource","msg":"default","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.485Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-v8ndd","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:25.501Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","namespacedName":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.501Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"fc4bfd24-710f-428f-b643-11bebf0972c2","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.501Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","mode":"IntegratedOAuth","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.501Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.501Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.501Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.501Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.501Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e"}
{"level":"info","ts":"2026-06-10T10:39:25.501Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e"}
{"level":"info","ts":"2026-06-10T10:39:25.502Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"46fa5a5c-5890-475e-885f-b82c14ed6eb0","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.502Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"46fa5a5c-5890-475e-885f-b82c14ed6eb0"}
{"level":"info","ts":"2026-06-10T10:39:25.502Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"46fa5a5c-5890-475e-885f-b82c14ed6eb0","RayCluster":"long-running-v8ndd","State":""}
{"level":"info","ts":"2026-06-10T10:39:25.502Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"6f1fc809-3e61-4760-a853-56dfad2935ab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:25.502Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"6f1fc809-3e61-4760-a853-56dfad2935ab","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.502Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"6f1fc809-3e61-4760-a853-56dfad2935ab","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.502Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"6f1fc809-3e61-4760-a853-56dfad2935ab"}
{"level":"info","ts":"2026-06-10T10:39:25.502Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"6f1fc809-3e61-4760-a853-56dfad2935ab","RayCluster":"long-running-v8ndd","State":""}
{"level":"info","ts":"2026-06-10T10:39:25.507Z","logger":"raycluster-resource","msg":"default","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.507Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-v8ndd","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:25.510Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","name":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:25.510Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","error":"head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"error","ts":"2026-06-10T10:39:25.510Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.510Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:25.510Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.510Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.510Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.510Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:39:25.510Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","name":"long-running-v8ndd","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:25.512Z","msg":"Self-signed issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.512Z","msg":"Creating CA certificate for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.521Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"9c874f09-6b84-4d8f-a704-9326d6ec55fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:25.521Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"9c874f09-6b84-4d8f-a704-9326d6ec55fc","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.521Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"9c874f09-6b84-4d8f-a704-9326d6ec55fc","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.521Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"9c874f09-6b84-4d8f-a704-9326d6ec55fc"}
{"level":"info","ts":"2026-06-10T10:39:25.521Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"9c874f09-6b84-4d8f-a704-9326d6ec55fc","RayCluster":"long-running-v8ndd","State":""}
{"level":"info","ts":"2026-06-10T10:39:25.521Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"fc4bfd24-710f-428f-b643-11bebf0972c2","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.521Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"23a6c8a0-c2f8-4e64-afdf-0ffcdca40862","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.526Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","name":"long-running-v8ndd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-v8ndd\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"long-running-v8ndd"}}
{"level":"info","ts":"2026-06-10T10:39:25.526Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e"}
{"level":"error","ts":"2026-06-10T10:39:25.526Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b76a8188-075c-406f-9969-33922bee058e","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.526Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607"}
{"level":"info","ts":"2026-06-10T10:39:25.526Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607"}
{"level":"info","ts":"2026-06-10T10:39:25.526Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:25.526Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","error":"head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"error","ts":"2026-06-10T10:39:25.526Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.526Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:25.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.527Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","oldReadyWorkerReplicas":0,"newReadyWorkerReplicas":0,"oldAvailableWorkerReplicas":0,"newAvailableWorkerReplicas":0,"oldDesiredWorkerReplicas":0,"newDesiredWorkerReplicas":1,"oldMinWorkerReplicas":0,"newMinWorkerReplicas":1,"oldMaxWorkerReplicas":0,"newMaxWorkerReplicas":1}
{"level":"info","ts":"2026-06-10T10:39:25.527Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","name":"long-running-v8ndd","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:25.528Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa"}
{"level":"info","ts":"2026-06-10T10:39:25.528Z","msg":"CA Certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","rayCluster":"long-running-v8ndd","secret":"ca-secret"}
{"level":"info","ts":"2026-06-10T10:39:25.528Z","msg":"Creating CA issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.530Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","name":"long-running-v8ndd-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:39:25.535Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","name":"kuberay-gateway-access","namespace":"test-ns-k92wt","operation":"created"}
{"level":"error","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"3049259f-e7bc-4302-b621-8b90bf9a3808","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"3049259f-e7bc-4302-b621-8b90bf9a3808"}
{"level":"error","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"3049259f-e7bc-4302-b621-8b90bf9a3808","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"23a6c8a0-c2f8-4e64-afdf-0ffcdca40862","name":"long-running-v8ndd-head"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","msg":"CA issuer created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607"}
{"level":"error","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"dfee524d-6ea3-44f2-a6cf-72cacf6f0607","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077","error":"head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"ace376da-9fda-4d25-b5a9-8478f8925aa2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"ace376da-9fda-4d25-b5a9-8478f8925aa2","name":"long-running-v8ndd"}
{"level":"error","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"ace376da-9fda-4d25-b5a9-8478f8925aa2","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.599Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.600Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"ace376da-9fda-4d25-b5a9-8478f8925aa2"}
{"level":"info","ts":"2026-06-10T10:39:25.600Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077"}
{"level":"error","ts":"2026-06-10T10:39:25.600Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"4f8789d5-163d-4aff-94c6-f43143ffc077","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"ace376da-9fda-4d25-b5a9-8478f8925aa2","RayCluster":"long-running-v8ndd","State":""}
{"level":"info","ts":"2026-06-10T10:39:25.609Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","name":"test-ns-k92wt-long-running-v8ndd","operation":"created"}
{"level":"info","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355"}
{"level":"info","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355"}
{"level":"info","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355","error":"head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"error","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355"}
{"level":"error","ts":"2026-06-10T10:39:25.610Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a01ad360-4f8a-4f21-8fa9-6efcfd045355","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.614Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"23a6c8a0-c2f8-4e64-afdf-0ffcdca40862","name":"long-running-v8ndd-workers"}
{"level":"info","ts":"2026-06-10T10:39:25.614Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"23a6c8a0-c2f8-4e64-afdf-0ffcdca40862","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.614Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"adc2af1e-967f-4910-8225-90a96f2f6b0d","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.614Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa"}
{"level":"info","ts":"2026-06-10T10:39:25.614Z","msg":"Ray head certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","rayCluster":"long-running-v8ndd","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:39:25.616Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","name":"kube-rbac-proxy-config-long-running-v8ndd","operation":"created"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","msg":"spec.privateKey.rotationPolicy: In cert-manager >= v1.18.0, the default value changed from `Never` to `Always`.","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945","error":"head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"error","ts":"2026-06-10T10:39:25.699Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"error","ts":"2026-06-10T10:39:25.699Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-v8ndd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1"}
{"level":"error","ts":"2026-06-10T10:39:25.699Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"034d5f53-4357-4820-9d30-35758bf340c1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-v8ndd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d9d9b539-18e8-4887-804a-42dd960afa13","namespacedName":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","msg":"Ray worker certificate created successfully","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","rayCluster":"long-running-v8ndd","podIPs":[]}
{"level":"error","ts":"2026-06-10T10:39:25.700Z","msg":"Failed to check certificate readiness","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-v8ndd\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:220\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.700Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa"}
{"level":"error","ts":"2026-06-10T10:39:25.700Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"11f2fe58-0377-444c-8f35-e08de2baaeaa","error":"Certificate.cert-manager.io \"ray-worker-cert-long-running-v8ndd\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.700Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"aa664a8d-19ce-49e7-97ae-e787fa8a5144","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d9d9b539-18e8-4887-804a-42dd960afa13","mode":"IntegratedOAuth","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.700Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d9d9b539-18e8-4887-804a-42dd960afa13","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.700Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"aa664a8d-19ce-49e7-97ae-e787fa8a5144"}
{"level":"info","ts":"2026-06-10T10:39:25.699Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.700Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945"}
{"level":"error","ts":"2026-06-10T10:39:25.700Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"10831df0-acc0-4587-967d-62c96c6e3945","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.704Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"adc2af1e-967f-4910-8225-90a96f2f6b0d","name":"long-running-v8ndd-head"}
{"level":"info","ts":"2026-06-10T10:39:25.705Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"32585767-992f-4051-ae35-1478cc12c302","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.705Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"32585767-992f-4051-ae35-1478cc12c302"}
{"level":"info","ts":"2026-06-10T10:39:25.711Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d9d9b539-18e8-4887-804a-42dd960afa13","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.711Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"c1dc1372-99cb-4ebd-bb74-2f0ce6ae93bb","namespacedName":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.711Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"c1dc1372-99cb-4ebd-bb74-2f0ce6ae93bb","mode":"IntegratedOAuth","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.711Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"c1dc1372-99cb-4ebd-bb74-2f0ce6ae93bb","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.712Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"a9fe292b-7c75-4e91-9dd4-81e8c45c83da","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.712Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"a9fe292b-7c75-4e91-9dd4-81e8c45c83da"}
{"level":"info","ts":"2026-06-10T10:39:25.712Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"3056cbdc-5103-49d7-affa-1cff18eea5c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:25.712Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"3056cbdc-5103-49d7-affa-1cff18eea5c1","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.712Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"3056cbdc-5103-49d7-affa-1cff18eea5c1","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.712Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"3056cbdc-5103-49d7-affa-1cff18eea5c1"}
{"level":"info","ts":"2026-06-10T10:39:25.712Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"3056cbdc-5103-49d7-affa-1cff18eea5c1","RayCluster":"long-running-v8ndd","State":""}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe"}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe"}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe","error":"head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"adc2af1e-967f-4910-8225-90a96f2f6b0d","name":"long-running-v8ndd-workers"}
{"level":"error","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"adc2af1e-967f-4910-8225-90a96f2f6b0d","cluster":"long-running-v8ndd"}
{"level":"error","ts":"2026-06-10T10:39:25.799Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"c1dc1372-99cb-4ebd-bb74-2f0ce6ae93bb","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-v8ndd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:186\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"c1dc1372-99cb-4ebd-bb74-2f0ce6ae93bb"}
{"level":"error","ts":"2026-06-10T10:39:25.799Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"c1dc1372-99cb-4ebd-bb74-2f0ce6ae93bb","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-v8ndd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"bfe8d3d1-726c-4acd-829c-aa8123a5ca78","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe"}
{"level":"error","ts":"2026-06-10T10:39:25.799Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2cdf5823-f1b9-4ac6-b78a-e6bbf9e892fe","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.805Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"0b364a29-c2f4-4657-92d4-c864a6856448","namespacedName":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:25.805Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"0b364a29-c2f4-4657-92d4-c864a6856448","mode":"IntegratedOAuth","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.805Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"0b364a29-c2f4-4657-92d4-c864a6856448","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.810Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"bfe8d3d1-726c-4acd-829c-aa8123a5ca78","name":"long-running-v8ndd-head"}
{"level":"info","ts":"2026-06-10T10:39:25.815Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"0b364a29-c2f4-4657-92d4-c864a6856448","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.822Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"bfe8d3d1-726c-4acd-829c-aa8123a5ca78","name":"long-running-v8ndd-workers"}
{"level":"info","ts":"2026-06-10T10:39:25.822Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"bfe8d3d1-726c-4acd-829c-aa8123a5ca78","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4"}
{"level":"info","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4"}
{"level":"info","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4","error":"head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"error","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4"}
{"level":"error","ts":"2026-06-10T10:39:25.960Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aae89b89-b261-4db3-a1a8-e92963498fd4","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af"}
{"level":"info","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af"}
{"level":"info","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af","error":"head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"error","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af"}
{"level":"error","ts":"2026-06-10T10:39:26.281Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"8715ad8d-fb9f-4910-b33c-0fa1b2d2a2af","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca"}
{"level":"info","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca"}
{"level":"info","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready yet, requeuing","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca","error":"head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"error","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found"}
{"level":"info","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca"}
{"level":"error","ts":"2026-06-10T10:39:26.923Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"964cfefd-f5e1-4791-9c04-c02102ea80ca","error":"mTLS secrets not ready: head certificate secret ray-head-secret-long-running-v8ndd not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3"}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3"}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3"}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3"}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","mode":"IntegratedOAuth","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","cluster":"long-running-v8ndd","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","cluster":"long-running-v8ndd","authType":"IntegratedOAuth","serviceAccount":"long-running-v8ndd-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-v8ndd","ray.io/group":"headgroup","ray.io/identifier":"long-running-v8ndd-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:39:28.204Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","enabled":true}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","name":"long-running-v8ndd-head-bj7kx"}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3"}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:28.237Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","enabled":true}
{"level":"info","ts":"2026-06-10T10:39:28.263Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","name":"long-running-v8ndd-small-group-worker-885kh"}
{"level":"info","ts":"2026-06-10T10:39:28.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.263Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.263Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-v8ndd-head-svc"},"newHead":{"podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"}}
{"level":"info","ts":"2026-06-10T10:39:28.263Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e939907d-1a39-4ffc-84ce-518f1d8464c3","name":"long-running-v8ndd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"a2a32762-3129-4a20-94bb-8a536163f8e8","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"ca77e379-49cb-4277-9087-383b405197a7","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"e675443b-9bae-469d-8f9f-7d3beb6421e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"e675443b-9bae-469d-8f9f-7d3beb6421e4","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"ca77e379-49cb-4277-9087-383b405197a7","rayCluster":"long-running-v8ndd","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"serviceName":"long-running-v8ndd-head-svc"},"newHead":{"podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"}}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"e675443b-9bae-469d-8f9f-7d3beb6421e4","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:28.280Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","name":"long-running-v8ndd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:28.281Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"e675443b-9bae-469d-8f9f-7d3beb6421e4"}
{"level":"info","ts":"2026-06-10T10:39:28.281Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"e675443b-9bae-469d-8f9f-7d3beb6421e4","RayCluster":"long-running-v8ndd","State":""}
{"level":"info","ts":"2026-06-10T10:39:28.293Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"a2a32762-3129-4a20-94bb-8a536163f8e8","name":"long-running-v8ndd-head"}
{"level":"info","ts":"2026-06-10T10:39:28.294Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","name":"long-running-v8ndd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-v8ndd\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-k92wt","name":"long-running-v8ndd"}}
{"level":"info","ts":"2026-06-10T10:39:28.294Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e"}
{"level":"error","ts":"2026-06-10T10:39:28.294Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"45941367-62c0-4785-ba45-e6672f830d0e","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-v8ndd\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:28.299Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295"}
{"level":"info","ts":"2026-06-10T10:39:28.299Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295"}
{"level":"info","ts":"2026-06-10T10:39:28.299Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:28.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:28.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:28.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:28.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.300Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:39:28.300Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c3666ecd-94aa-4325-8419-31c8e61ef295","name":"long-running-v8ndd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:28.307Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"a2a32762-3129-4a20-94bb-8a536163f8e8","name":"long-running-v8ndd-workers"}
{"level":"info","ts":"2026-06-10T10:39:28.307Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"a2a32762-3129-4a20-94bb-8a536163f8e8","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.399Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"5790bfdd-f4d5-4dcd-84b2-e62c6e32d7e2","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a"}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a"}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:28.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.401Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.499Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.499Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"7e8d78b6-6b11-47a7-a9b5-274a34bdb4ec","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2a15bda6-f367-488c-a8cc-9647c79ed640","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2a15bda6-f367-488c-a8cc-9647c79ed640","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2a15bda6-f367-488c-a8cc-9647c79ed640","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e8eb435e-61a8-43f8-bbe7-3858fcd0576a","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"7e8d78b6-6b11-47a7-a9b5-274a34bdb4ec","rayCluster":"long-running-v8ndd","podIPs":[]}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2a15bda6-f367-488c-a8cc-9647c79ed640"}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2a15bda6-f367-488c-a8cc-9647c79ed640","RayCluster":"long-running-v8ndd","State":""}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"8d197ce2-7965-4f7a-bff4-4e997fc1c831","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"8d197ce2-7965-4f7a-bff4-4e997fc1c831","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"8d197ce2-7965-4f7a-bff4-4e997fc1c831","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"8d197ce2-7965-4f7a-bff4-4e997fc1c831"}
{"level":"info","ts":"2026-06-10T10:39:28.500Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"8d197ce2-7965-4f7a-bff4-4e997fc1c831","RayCluster":"long-running-v8ndd","State":""}
{"level":"info","ts":"2026-06-10T10:39:28.506Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"5790bfdd-f4d5-4dcd-84b2-e62c6e32d7e2","name":"long-running-v8ndd-head"}
{"level":"info","ts":"2026-06-10T10:39:28.519Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"5790bfdd-f4d5-4dcd-84b2-e62c6e32d7e2","name":"long-running-v8ndd-workers"}
{"level":"info","ts":"2026-06-10T10:39:28.519Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"5790bfdd-f4d5-4dcd-84b2-e62c6e32d7e2","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:28.703Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574"}
{"level":"info","ts":"2026-06-10T10:39:28.703Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574"}
{"level":"info","ts":"2026-06-10T10:39:28.703Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:28.704Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"7116036b-1b7b-4a93-ac03-de1398f3d574","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270"}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270"}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:28.730Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"99e9a9c6-7f9e-45ee-ad10-f60a95f87270","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e"}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e"}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:29.160Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"newHead":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"}}
{"level":"info","ts":"2026-06-10T10:39:29.161Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"de697118-1395-4b8c-a7bd-330a9026e34e","name":"long-running-v8ndd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:29.172Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"847d9bce-7961-4661-bd46-34a3bc8a706f","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:29.172Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"815599a9-26f1-40d6-be0d-6defcd6168ef","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:29.172Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"815599a9-26f1-40d6-be0d-6defcd6168ef","pod":"long-running-v8ndd-head-bj7kx","ip":"10.134.0.34"}
{"level":"info","ts":"2026-06-10T10:39:29.172Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"11fdc7db-83dd-47df-ab2f-9d3cc028546a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:29.172Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"11fdc7db-83dd-47df-ab2f-9d3cc028546a","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:29.172Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"11fdc7db-83dd-47df-ab2f-9d3cc028546a","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:29.173Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"11fdc7db-83dd-47df-ab2f-9d3cc028546a"}
{"level":"info","ts":"2026-06-10T10:39:29.173Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"11fdc7db-83dd-47df-ab2f-9d3cc028546a","RayCluster":"long-running-v8ndd","State":""}
{"level":"info","ts":"2026-06-10T10:39:29.183Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"847d9bce-7961-4661-bd46-34a3bc8a706f","name":"long-running-v8ndd-head"}
{"level":"info","ts":"2026-06-10T10:39:29.184Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"815599a9-26f1-40d6-be0d-6defcd6168ef","certificate":"ray-head-cert-long-running-v8ndd","dnsNames":["long-running-v8ndd-head-svc","localhost","long-running-v8ndd-head-svc.test-ns-k92wt.svc","long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:39:29.195Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"815599a9-26f1-40d6-be0d-6defcd6168ef","certificate":"ray-worker-cert-long-running-v8ndd","dnsNames":["long-running-v8ndd-worker-svc","localhost","long-running-v8ndd-small-group","long-running-v8ndd-worker-svc.test-ns-k92wt.svc","long-running-v8ndd-worker-svc.test-ns-k92wt.svc.cluster.local","*.long-running-v8ndd-worker-svc.test-ns-k92wt.svc","*.long-running-v8ndd-worker-svc.test-ns-k92wt.svc.cluster.local","*-worker-*.test-ns-k92wt.svc","*-worker-*.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:39:29.195Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"815599a9-26f1-40d6-be0d-6defcd6168ef","rayCluster":"long-running-v8ndd","podIPs":["10.134.0.34"]}
{"level":"info","ts":"2026-06-10T10:39:29.197Z","logger":"networkpolicy-controller","msg":"Successfully updated NetworkPolicy","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"847d9bce-7961-4661-bd46-34a3bc8a706f","name":"long-running-v8ndd-workers"}
{"level":"info","ts":"2026-06-10T10:39:29.197Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"847d9bce-7961-4661-bd46-34a3bc8a706f","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c"}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c"}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:29.753Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"a05f294e-7226-476b-b1c4-c80c96b02c6c","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad"}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad"}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:30.400Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b3aa7e38-838e-4965-b60a-477bb54cb0ad","seconds":300}
{"level":"error","ts":"2026-06-10T10:39:30.720Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"161a8949-a43a-4b5d-8a0e-2a4cc1822f83","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:30.720Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"161a8949-a43a-4b5d-8a0e-2a4cc1822f83"}
{"level":"error","ts":"2026-06-10T10:39:30.720Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"161a8949-a43a-4b5d-8a0e-2a4cc1822f83","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:31.501Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"0dc311eb-388e-458d-950b-6ac2ac192a1d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:31.501Z","logger":"controllers.RayJob","msg":"The RayJob has passed the activeDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"0dc311eb-388e-458d-950b-6ac2ac192a1d","StartTime":"2026-06-10 10:39:25 +0000 UTC","ActiveDeadlineSeconds":5}
{"level":"info","ts":"2026-06-10T10:39:31.501Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"0dc311eb-388e-458d-950b-6ac2ac192a1d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-pvbmz","rayClusterName":"long-running-v8ndd","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:39:25Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-pvbmz","rayClusterName":"long-running-v8ndd","jobDeploymentStatus":"Failed","reason":"DeadlineExceeded","message":"The RayJob has passed the activeDeadlineSeconds. StartTime: 2026-06-10 10:39:25 +0000 UTC. ActiveDeadlineSeconds: 5","startTime":"2026-06-10T10:39:25Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:39:31.501Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"0dc311eb-388e-458d-950b-6ac2ac192a1d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-06-10T10:39:31.513Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2c4c30cf-bb78-4206-bd9d-cf085d992bad","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:31.513Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2c4c30cf-bb78-4206-bd9d-cf085d992bad","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-10 10:39:31 +0000 UTC","Now":"2026-06-10T10:39:31.513Z","ShutdownTime":"2026-06-10T10:49:31.000Z"}
{"level":"info","ts":"2026-06-10T10:39:31.513Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2c4c30cf-bb78-4206-bd9d-cf085d992bad","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:39:31.514Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2c4c30cf-bb78-4206-bd9d-cf085d992bad","seconds":601}
{"level":"info","ts":"2026-06-10T10:39:32.515Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-k92wt"},"reconcileID":"04db2a7b-7701-48e9-a62e-4edf9fe5191e","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-06-10T10:39:32.548Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"managed-externally","namespace":"test-ns-k92wt"},"reconcileID":"7e9f6bf0-e32c-4043-8dfb-0a7ff1062649"}
{"level":"info","ts":"2026-06-10T10:39:32.636Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-ccmjv"},"reconcileID":"17389429-8925-46ab-891b-f02ccc8d5708","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b"}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b"}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:34.180Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"0bb24973-0519-4309-9add-4058306afd4b","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:34.514Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"854ccf4b-6e00-470b-bdf9-081144f7115d","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:34.514Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"854ccf4b-6e00-470b-bdf9-081144f7115d","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-10 10:39:31 +0000 UTC","Now":"2026-06-10T10:39:34.514Z","ShutdownTime":"2026-06-10T10:49:31.000Z"}
{"level":"info","ts":"2026-06-10T10:39:34.514Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"854ccf4b-6e00-470b-bdf9-081144f7115d","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:39:34.514Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"854ccf4b-6e00-470b-bdf9-081144f7115d","seconds":598}
{"level":"info","ts":"2026-06-10T10:39:35.187Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd"}
{"level":"info","ts":"2026-06-10T10:39:35.187Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd"}
{"level":"info","ts":"2026-06-10T10:39:35.187Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}]}
{"level":"info","ts":"2026-06-10T10:39:35.188Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd2dc404-8468-4863-88ce-41aa24fc67dd","name":"long-running-v8ndd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head kube-rbac-proxy]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:35.199Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"54f65571-8eaf-4d75-ab1c-4ac8a2c18383","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:35.199Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"ca093b60-b1d1-481e-825a-f5b4457e6f2b","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:35.200Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"54f65571-8eaf-4d75-ab1c-4ac8a2c18383","pod":"long-running-v8ndd-head-bj7kx","ip":"10.134.0.34"}
{"level":"info","ts":"2026-06-10T10:39:35.200Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"54f65571-8eaf-4d75-ab1c-4ac8a2c18383","pod":"long-running-v8ndd-small-group-worker-885kh","ip":"10.132.0.50"}
{"level":"info","ts":"2026-06-10T10:39:35.200Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"4ef2d3e9-4992-4759-bdf5-cd7a0147acc3","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:35.200Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"4ef2d3e9-4992-4759-bdf5-cd7a0147acc3","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-10 10:39:31 +0000 UTC","Now":"2026-06-10T10:39:35.200Z","ShutdownTime":"2026-06-10T10:49:31.000Z"}
{"level":"info","ts":"2026-06-10T10:39:35.200Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"4ef2d3e9-4992-4759-bdf5-cd7a0147acc3","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:39:35.200Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"4ef2d3e9-4992-4759-bdf5-cd7a0147acc3","seconds":597}
{"level":"info","ts":"2026-06-10T10:39:35.204Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"ca093b60-b1d1-481e-825a-f5b4457e6f2b","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:35.223Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"54f65571-8eaf-4d75-ab1c-4ac8a2c18383","certificate":"ray-head-cert-long-running-v8ndd","dnsNames":["long-running-v8ndd-head-svc","localhost","long-running-v8ndd-head-svc.test-ns-k92wt.svc","long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.34","10.132.0.50","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:39:35.257Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"54f65571-8eaf-4d75-ab1c-4ac8a2c18383","certificate":"ray-worker-cert-long-running-v8ndd","dnsNames":["long-running-v8ndd-worker-svc","localhost","long-running-v8ndd-small-group","long-running-v8ndd-worker-svc.test-ns-k92wt.svc","long-running-v8ndd-worker-svc.test-ns-k92wt.svc.cluster.local","*.long-running-v8ndd-worker-svc.test-ns-k92wt.svc","*.long-running-v8ndd-worker-svc.test-ns-k92wt.svc.cluster.local","*-worker-*.test-ns-k92wt.svc","*-worker-*.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.34","10.132.0.50","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:39:35.257Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"54f65571-8eaf-4d75-ab1c-4ac8a2c18383"}
{"level":"info","ts":"2026-06-10T10:39:35.651Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"109fab2c-a377-4607-97ea-32b619d0606b","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:35.652Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"109fab2c-a377-4607-97ea-32b619d0606b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:39:35.660Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"109fab2c-a377-4607-97ea-32b619d0606b"}
{"level":"info","ts":"2026-06-10T10:39:35.660Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"109fab2c-a377-4607-97ea-32b619d0606b","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:39:35.660Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"109fab2c-a377-4607-97ea-32b619d0606b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-9wf7b","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:39:35.660Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"109fab2c-a377-4607-97ea-32b619d0606b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:39:35.669Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:35.669Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:35.669Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:39:35.669Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","head service name":"raycluster-head-svc","namespace":"test-ns-ccmjv"}
{"level":"info","ts":"2026-06-10T10:39:35.669Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","head service URL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:39:35.669Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a"}
{"level":"info","ts":"2026-06-10T10:39:35.669Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","counter-9wf7b",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","--no-wait","--runtime-env-json","\"{\\\"env_vars\\\":{\\\"counter_name\\\":\\\"test_counter\\\"}}\"","--submission-id","counter-9wf7b","--","python /home/ray/jobs/counter.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","--follow","counter-9wf7b"]}
{"level":"info","ts":"2026-06-10T10:39:35.677Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-10T10:39:35.677Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:35.677Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-9wf7b","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:39:35.677Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1c6ace2a-ba25-4827-a4c7-8b665313610a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:39:35.686Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:35.686Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:35.686Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:39:35.686Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","head service name":"raycluster-head-svc","namespace":"test-ns-ccmjv"}
{"level":"info","ts":"2026-06-10T10:39:35.686Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","head service URL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:39:35.686Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-06-10T10:39:35.686Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:35.686Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-9wf7b","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:39:35.686Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:39:35.690Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-06-10T10:39:35.690Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d"}
{"level":"error","ts":"2026-06-10T10:39:35.691Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"5f24c8c7-6e49-4f7d-94c0-77d06150e55d","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.691Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"cc13e3e0-0451-45e2-8dd1-63c8c27cc5ef","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:35.691Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"cc13e3e0-0451-45e2-8dd1-63c8c27cc5ef","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:35.691Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"cc13e3e0-0451-45e2-8dd1-63c8c27cc5ef","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:35.699Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"cc13e3e0-0451-45e2-8dd1-63c8c27cc5ef","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.699Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"cc13e3e0-0451-45e2-8dd1-63c8c27cc5ef"}
{"level":"error","ts":"2026-06-10T10:39:35.699Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"cc13e3e0-0451-45e2-8dd1-63c8c27cc5ef","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.699Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"eff3e714-6663-4882-bbbe-621a58360f73","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:35.699Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"eff3e714-6663-4882-bbbe-621a58360f73","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:35.699Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"eff3e714-6663-4882-bbbe-621a58360f73","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:35.703Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"eff3e714-6663-4882-bbbe-621a58360f73","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.703Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"eff3e714-6663-4882-bbbe-621a58360f73"}
{"level":"error","ts":"2026-06-10T10:39:35.703Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"eff3e714-6663-4882-bbbe-621a58360f73","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.709Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"42c005cf-323d-467e-9f37-df2a94c151c5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:35.709Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"42c005cf-323d-467e-9f37-df2a94c151c5","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:35.709Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"42c005cf-323d-467e-9f37-df2a94c151c5","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:35.714Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"42c005cf-323d-467e-9f37-df2a94c151c5","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.714Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"42c005cf-323d-467e-9f37-df2a94c151c5"}
{"level":"error","ts":"2026-06-10T10:39:35.714Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"42c005cf-323d-467e-9f37-df2a94c151c5","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.754Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1d899986-78e0-450f-b199-81064e6d986f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:35.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1d899986-78e0-450f-b199-81064e6d986f","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:35.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1d899986-78e0-450f-b199-81064e6d986f","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:35.760Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1d899986-78e0-450f-b199-81064e6d986f","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.760Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1d899986-78e0-450f-b199-81064e6d986f"}
{"level":"error","ts":"2026-06-10T10:39:35.760Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"1d899986-78e0-450f-b199-81064e6d986f","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.840Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"28c98b7a-5736-47df-b030-a9ee04cc9801","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:35.840Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"28c98b7a-5736-47df-b030-a9ee04cc9801","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:35.840Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"28c98b7a-5736-47df-b030-a9ee04cc9801","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:35.846Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"28c98b7a-5736-47df-b030-a9ee04cc9801","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:35.846Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"28c98b7a-5736-47df-b030-a9ee04cc9801"}
{"level":"error","ts":"2026-06-10T10:39:35.846Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"28c98b7a-5736-47df-b030-a9ee04cc9801","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:36.006Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"7a86c3f2-666e-4a7b-bdd6-d5a7d7ea6ee9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:36.006Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"7a86c3f2-666e-4a7b-bdd6-d5a7d7ea6ee9","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:36.006Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"7a86c3f2-666e-4a7b-bdd6-d5a7d7ea6ee9","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:36.011Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"7a86c3f2-666e-4a7b-bdd6-d5a7d7ea6ee9","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:36.012Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"7a86c3f2-666e-4a7b-bdd6-d5a7d7ea6ee9"}
{"level":"error","ts":"2026-06-10T10:39:36.012Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"7a86c3f2-666e-4a7b-bdd6-d5a7d7ea6ee9","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:36.332Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"33c20ea3-bda0-440f-8d4f-c2b43a65ef99","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:36.332Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"33c20ea3-bda0-440f-8d4f-c2b43a65ef99","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:36.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"33c20ea3-bda0-440f-8d4f-c2b43a65ef99","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:36.338Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"33c20ea3-bda0-440f-8d4f-c2b43a65ef99","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:36.338Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"33c20ea3-bda0-440f-8d4f-c2b43a65ef99"}
{"level":"error","ts":"2026-06-10T10:39:36.338Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"33c20ea3-bda0-440f-8d4f-c2b43a65ef99","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:36.979Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"78b6ca9f-94fb-482b-a54b-05a57c01127e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:36.979Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"78b6ca9f-94fb-482b-a54b-05a57c01127e","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:36.979Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"78b6ca9f-94fb-482b-a54b-05a57c01127e","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:36.984Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"78b6ca9f-94fb-482b-a54b-05a57c01127e","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:36.984Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"78b6ca9f-94fb-482b-a54b-05a57c01127e"}
{"level":"error","ts":"2026-06-10T10:39:36.984Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"78b6ca9f-94fb-482b-a54b-05a57c01127e","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06"}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06"}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:37.200Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"b77537b6-40df-4ef0-8303-276803f7ba06","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca"}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca"}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:37.652Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c485784d-1f9b-4a4d-9da3-e598a4d675ca","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f"}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f"}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:37.657Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"2949c5e5-58ac-484b-999e-8aa62b09437f","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f"}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f"}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","podName":"long-running-v8ndd-small-group-worker-885kh","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:37.676Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"3772bd98-c067-4f9f-af2d-cec77607e21f","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:37.678Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727"}
{"level":"info","ts":"2026-06-10T10:39:37.678Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727"}
{"level":"info","ts":"2026-06-10T10:39:37.678Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:37.679Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e885448a-8695-478b-b0b6-900bd1a2c727","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:38.209Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"932c76dd-d32d-4e94-8d52-04822613839e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:38.209Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"932c76dd-d32d-4e94-8d52-04822613839e","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:38.209Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"932c76dd-d32d-4e94-8d52-04822613839e","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:38.214Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"932c76dd-d32d-4e94-8d52-04822613839e","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.214Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"932c76dd-d32d-4e94-8d52-04822613839e"}
{"level":"error","ts":"2026-06-10T10:39:38.214Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"932c76dd-d32d-4e94-8d52-04822613839e","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6"}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6"}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-v8ndd-small-group-worker-885kh. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:38.231Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"36647fef-9856-433a-baa7-fb70b2cd25b6","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:38.265Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ea7f9da8-68fd-42dc-889a-25c0fb528b50","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:38.265Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ea7f9da8-68fd-42dc-889a-25c0fb528b50","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:38.265Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ea7f9da8-68fd-42dc-889a-25c0fb528b50","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:38.270Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ea7f9da8-68fd-42dc-889a-25c0fb528b50","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.270Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ea7f9da8-68fd-42dc-889a-25c0fb528b50"}
{"level":"error","ts":"2026-06-10T10:39:38.270Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ea7f9da8-68fd-42dc-889a-25c0fb528b50","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.818Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde"}
{"level":"info","ts":"2026-06-10T10:39:38.818Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde"}
{"level":"info","ts":"2026-06-10T10:39:38.818Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:38.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:38.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:38.818Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde","worker Pod":"long-running-v8ndd-small-group-worker-885kh","shouldDelete":true,"reason":"The worker Pod long-running-v8ndd-small-group-worker-885kh status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-10T10:39:38.821Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.821Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-06-10T10:39:38.821Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.821Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde"}
{"level":"error","ts":"2026-06-10T10:39:38.822Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"e3994743-371d-4b68-87e5-1c0dbd627dde","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.826Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25"}
{"level":"info","ts":"2026-06-10T10:39:38.826Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25"}
{"level":"info","ts":"2026-06-10T10:39:38.826Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"ef75fecc-37ed-472f-8b67-d0a4b49b0e25","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b"}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b"}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-06-10T10:39:38.827Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"c14ec506-7faf-4762-b5ba-c1da68cd891b","seconds":300}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37"}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37"}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37"}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.830Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:38.834Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.834Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:38.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.834Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:39:38Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}]}
{"level":"info","ts":"2026-06-10T10:39:38.834Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","name":"long-running-v8ndd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:39:38Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:38.846Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37"}
{"level":"error","ts":"2026-06-10T10:39:38.846Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bc99d831-9d5d-40ca-9e04-620668f97f37","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.847Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"cd3566e0-3c4e-48be-9022-e5ef980caec9","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:38.847Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"88387730-56c5-4f28-ac96-ab0108ccc4bd","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:38.847Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"88387730-56c5-4f28-ac96-ab0108ccc4bd","pod":"long-running-v8ndd-head-bj7kx","ip":"10.134.0.34"}
{"level":"info","ts":"2026-06-10T10:39:38.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"32e7568b-1320-4160-b61b-dc5fc15b7386","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:38.847Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"32e7568b-1320-4160-b61b-dc5fc15b7386","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-10 10:39:31 +0000 UTC","Now":"2026-06-10T10:39:38.847Z","ShutdownTime":"2026-06-10T10:49:31.000Z"}
{"level":"info","ts":"2026-06-10T10:39:38.847Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"32e7568b-1320-4160-b61b-dc5fc15b7386","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:39:38.847Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"32e7568b-1320-4160-b61b-dc5fc15b7386","seconds":594}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3"}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3"}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3"}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"cd3566e0-3c4e-48be-9022-e5ef980caec9","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:38.851Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:38.854Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.854Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:38.854Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.854Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3"}
{"level":"error","ts":"2026-06-10T10:39:38.854Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"bd2702e3-a967-47d7-a558-c0e262af58a3","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.861Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"88387730-56c5-4f28-ac96-ab0108ccc4bd","certificate":"ray-head-cert-long-running-v8ndd","dnsNames":["long-running-v8ndd-head-svc","localhost","long-running-v8ndd-head-svc.test-ns-k92wt.svc","long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8"}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8"}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8"}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.875Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:38.899Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.899Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:38.899Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.899Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8"}
{"level":"error","ts":"2026-06-10T10:39:38.899Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cb538939-a53d-4e28-9e16-3059e37479b8","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.914Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"88387730-56c5-4f28-ac96-ab0108ccc4bd","certificate":"ray-worker-cert-long-running-v8ndd","dnsNames":["long-running-v8ndd-worker-svc","localhost","long-running-v8ndd-small-group","long-running-v8ndd-worker-svc.test-ns-k92wt.svc","long-running-v8ndd-worker-svc.test-ns-k92wt.svc.cluster.local","*.long-running-v8ndd-worker-svc.test-ns-k92wt.svc","*.long-running-v8ndd-worker-svc.test-ns-k92wt.svc.cluster.local","*-worker-*.test-ns-k92wt.svc","*-worker-*.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["10.134.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:39:38.914Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"88387730-56c5-4f28-ac96-ab0108ccc4bd"}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308"}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308"}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308"}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.920Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:38.923Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.923Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:38.924Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.924Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308"}
{"level":"error","ts":"2026-06-10T10:39:38.924Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"24e0a38e-f790-4c35-b5dc-785113144308","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.964Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d"}
{"level":"info","ts":"2026-06-10T10:39:38.964Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d"}
{"level":"info","ts":"2026-06-10T10:39:38.964Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d"}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:38.965Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:38.968Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:38.968Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:38.969Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:38.999Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d"}
{"level":"error","ts":"2026-06-10T10:39:38.999Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"aa1a2c90-686f-4c22-9ec8-4b08b33c879d","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1"}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1"}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1"}
{"level":"info","ts":"2026-06-10T10:39:39.079Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:39.080Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:39.080Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:39.080Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:39.083Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:39.083Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:39.083Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:39.083Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1"}
{"level":"error","ts":"2026-06-10T10:39:39.083Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"65749d69-9f8c-4692-82fe-11d1bacb75b1","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8"}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8"}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8"}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:39.244Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:39.248Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:39.248Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:39.248Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:39.248Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8"}
{"level":"error","ts":"2026-06-10T10:39:39.248Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"943e3da0-0a4d-4c31-abf2-cfbc657f18b8","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5"}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5"}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5"}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:39.569Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:39.572Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:39.572Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:39.573Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:39.573Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5"}
{"level":"error","ts":"2026-06-10T10:39:39.573Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d96ee37d-463c-4b5a-8e7e-9fe238084ca5","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3"}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3"}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-v8ndd-head-bj7kx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3"}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:40.214Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:40.217Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:40.218Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:40.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","podName":"long-running-v8ndd-head-bj7kx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":2,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-06-10T10:39:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-06-10T10:39:40.218Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3"}
{"level":"error","ts":"2026-06-10T10:39:40.218Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"6ab5e8ad-7f9c-47cd-812d-4bb6b25eb2c3","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:39:40.961Z","logger":"controllers.RayJob","msg":"The RayJob spec is invalid","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"9b088c12-06cf-4a00-95f0-b69980b140fa","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:40.961Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"9b088c12-06cf-4a00-95f0-b69980b140fa"}
{"level":"error","ts":"2026-06-10T10:39:40.961Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"9b088c12-06cf-4a00-95f0-b69980b140fa","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type utils.RuntimeEnvType","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:41.222Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c"}
{"level":"info","ts":"2026-06-10T10:39:41.222Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c"}
{"level":"info","ts":"2026-06-10T10:39:41.222Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:41.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c","Found 1 head Pod":"long-running-v8ndd-head-bj7kx","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-06-10 10:39:34 +0000 UTC,FinishedAt:2026-06-10 10:39:41 +0000 UTC,ContainerID:cri-o://44667a099e1ae40a9a0e01f54c75bc409919e31a0ab56be21161cee858b7699a,}"}
{"level":"info","ts":"2026-06-10T10:39:41.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c","head Pod":"long-running-v8ndd-head-bj7kx","shouldDelete":true,"reason":"The head Pod long-running-v8ndd-head-bj7kx status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-06-10T10:39:41.225Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod long-running-v8ndd-head-bj7kx status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:41.225Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c","error":"The head Pod long-running-v8ndd-head-bj7kx status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-06-10T10:39:41.225Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-10T10:39:41.225Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c","old conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:39:38Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}],"new conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:39:38Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}]}
{"level":"info","ts":"2026-06-10T10:39:41.225Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c","name":"long-running-v8ndd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:38Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:39:38Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:39:38Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:41.239Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c"}
{"level":"error","ts":"2026-06-10T10:39:41.239Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cf6ee9b7-ee95-4325-b1ec-706fec952e5c","error":"The head Pod long-running-v8ndd-head-bj7kx status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:41.239Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"2e2c2233-b1ff-4603-94ca-bea05e334a6a","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:41.239Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"88fbd64b-05bc-4444-8047-7ab9494056fc","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:41.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"8913ad72-6675-476a-b345-ea6f4318cccd","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:41.239Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"8913ad72-6675-476a-b345-ea6f4318cccd","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-10 10:39:31 +0000 UTC","Now":"2026-06-10T10:39:41.239Z","ShutdownTime":"2026-06-10T10:49:31.000Z"}
{"level":"info","ts":"2026-06-10T10:39:41.239Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"8913ad72-6675-476a-b345-ea6f4318cccd","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:39:41.239Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"8913ad72-6675-476a-b345-ea6f4318cccd","seconds":591}
{"level":"info","ts":"2026-06-10T10:39:41.239Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"88fbd64b-05bc-4444-8047-7ab9494056fc","pod":"long-running-v8ndd-head-bj7kx","ip":"10.134.0.34"}
{"level":"info","ts":"2026-06-10T10:39:41.239Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"88fbd64b-05bc-4444-8047-7ab9494056fc","rayCluster":"long-running-v8ndd","podIPs":["10.134.0.34"]}
{"level":"info","ts":"2026-06-10T10:39:41.245Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"2e2c2233-b1ff-4603-94ca-bea05e334a6a","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211"}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211"}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211"}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:41.498Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:41.503Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:41.503Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:41.503Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-10T10:39:41.504Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211"}
{"level":"error","ts":"2026-06-10T10:39:41.504Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"cd85f9cc-1f3c-4f5b-91b7-8b76cb0f9211","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8"}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8"}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","creating worker for group":"small-group","index":0,"total":1}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for worker pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8"}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","nodeType":"worker","rayStartParams":{"address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","rayStartCmd":"ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","rayNodeType":"worker","generatedCmd":"ulimit -n 65536; ray start  --address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-06-10T10:39:41.533Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:41.538Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:41.538Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:41.538Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","podName":"long-running-v8ndd-head-bj7kx","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-06-10T10:39:41.538Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8"}
{"level":"error","ts":"2026-06-10T10:39:41.538Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"41ab52ba-ca8b-44d4-aaaf-e4bcfec3b0c8","error":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:41.543Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4"}
{"level":"info","ts":"2026-06-10T10:39:41.543Z","logger":"controllers.RayCluster","msg":"EnableIngress is false, cleaning up any existing Routes/Ingresses","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4"}
{"level":"info","ts":"2026-06-10T10:39:41.543Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","1 head service found":"long-running-v8ndd-head-svc"}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4"}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"AuthenticationReady condition is True and fresh, proceeding with pod creation","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","reason":"AuthenticationResourcesCreated","observedGeneration":1,"authMode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"mTLS is enabled, configuring mTLS for head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4"}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","mode":"IntegratedOAuth","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","cluster":"long-running-v8ndd","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","cluster":"long-running-v8ndd","authType":"IntegratedOAuth","serviceAccount":"long-running-v8ndd-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-v8ndd","ray.io/group":"headgroup","ray.io/identifier":"long-running-v8ndd-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","node-ip-address":"long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","rayNodeType":"head","generatedCmd":"ulimit -n 65536; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --node-ip-address=long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local  --num-cpus=2 "}
{"level":"info","ts":"2026-06-10T10:39:41.544Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","enabled":true}
{"level":"error","ts":"2026-06-10T10:39:41.550Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"FailedCreateHeadPod\npods \"long-running-v8ndd-head-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:318\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:131\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:41.550Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","error":"FailedCreateHeadPod\npods \"long-running-v8ndd-head-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}
{"level":"info","ts":"2026-06-10T10:39:41.550Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:41.550Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:41.550Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","filter labels":["test-ns-k92wt",{"ray.io/cluster":"long-running-v8ndd","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-06-10T10:39:41.550Z","logger":"controllers.RayCluster","msg":"inconsistentRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","oldEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"newEndpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"oldHead":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"newHead":{"serviceName":"long-running-v8ndd-head-svc"}}
{"level":"info","ts":"2026-06-10T10:39:41.550Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","name":"long-running-v8ndd","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.34","serviceIP":"10.134.0.34","podName":"long-running-v8ndd-head-bj7kx","serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:39:38Z","reason":"FailedCreateWorkerPod","message":"FailedCreateWorkerPod\npods \"long-running-v8ndd-small-group-worker-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:39:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-v8ndd-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:39:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:39:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"ReplicaFailure","status":"True","lastTransitionTime":"2026-06-10T10:39:38Z","reason":"FailedCreateHeadPod","message":"FailedCreateHeadPod\npods \"long-running-v8ndd-head-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-06-10T10:39:41.577Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"b7784a04-56ed-4c4d-9cdb-5cbc5a7058ba","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:41.577Z","logger":"networkpolicy-controller","msg":"Reconciling NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"7f955f19-f53c-40d6-8943-9a71c586c61d","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:41.577Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"114a95e6-6afe-4cae-9ec7-5890ccadf1ab","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:41.577Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"114a95e6-6afe-4cae-9ec7-5890ccadf1ab","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-10 10:39:31 +0000 UTC","Now":"2026-06-10T10:39:41.577Z","ShutdownTime":"2026-06-10T10:49:31.000Z"}
{"level":"info","ts":"2026-06-10T10:39:41.577Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"114a95e6-6afe-4cae-9ec7-5890ccadf1ab","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:39:41.577Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"114a95e6-6afe-4cae-9ec7-5890ccadf1ab","seconds":591}
{"level":"info","ts":"2026-06-10T10:39:41.599Z","logger":"networkpolicy-controller","msg":"Successfully reconciled NetworkPolicies for RayCluster","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"7f955f19-f53c-40d6-8943-9a71c586c61d","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:41.599Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4"}
{"level":"error","ts":"2026-06-10T10:39:41.599Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"d79c237b-6c55-4582-a478-2b722f78cec4","error":"FailedCreateHeadPod\npods \"long-running-v8ndd-head-\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:41.602Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"b7784a04-56ed-4c4d-9cdb-5cbc5a7058ba","certificate":"ray-head-cert-long-running-v8ndd","dnsNames":["long-running-v8ndd-head-svc","localhost","long-running-v8ndd-head-svc.test-ns-k92wt.svc","long-running-v8ndd-head-svc.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:39:41.622Z","msg":"Updated certificate SANs","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"b7784a04-56ed-4c4d-9cdb-5cbc5a7058ba","certificate":"ray-worker-cert-long-running-v8ndd","dnsNames":["long-running-v8ndd-worker-svc","localhost","long-running-v8ndd-small-group","long-running-v8ndd-worker-svc.test-ns-k92wt.svc","long-running-v8ndd-worker-svc.test-ns-k92wt.svc.cluster.local","*.long-running-v8ndd-worker-svc.test-ns-k92wt.svc","*.long-running-v8ndd-worker-svc.test-ns-k92wt.svc.cluster.local","*-worker-*.test-ns-k92wt.svc","*-worker-*.test-ns-k92wt.svc.cluster.local"],"ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-06-10T10:39:41.622Z","msg":"One or more certificates are not ready, requeuing","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"b7784a04-56ed-4c4d-9cdb-5cbc5a7058ba"}
{"level":"info","ts":"2026-06-10T10:39:43.390Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"facacb94-ed91-4166-9e49-94a70d7517e1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:43.391Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"facacb94-ed91-4166-9e49-94a70d7517e1","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:43.391Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"facacb94-ed91-4166-9e49-94a70d7517e1","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:43.396Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"facacb94-ed91-4166-9e49-94a70d7517e1","JobId":"counter-9wf7b","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:43.396Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"facacb94-ed91-4166-9e49-94a70d7517e1"}
{"level":"error","ts":"2026-06-10T10:39:43.396Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"facacb94-ed91-4166-9e49-94a70d7517e1","error":"Job counter-9wf7b does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"6e7aef73-e7c7-4cf9-9443-d1d9ad307695","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"855c8085-93bd-4cf0-8826-858c0de02eb9","namespacedName":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"6e7aef73-e7c7-4cf9-9443-d1d9ad307695","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"reconcileID":"386342fd-a9da-47fb-88fd-6641d504c963"}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"855c8085-93bd-4cf0-8826-858c0de02eb9","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"855c8085-93bd-4cf0-8826-858c0de02eb9","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"networkpolicy-controller","msg":"RayCluster is being deleted, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"333b2839-f239-42f9-a822-9d25cf8af70c"}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"94bccaef-960c-4449-8be2-ea928ead6ee2","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"94bccaef-960c-4449-8be2-ea928ead6ee2","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-10 10:39:31 +0000 UTC","Now":"2026-06-10T10:39:45.349Z","ShutdownTime":"2026-06-10T10:49:31.000Z"}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"94bccaef-960c-4449-8be2-ea928ead6ee2","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"94bccaef-960c-4449-8be2-ea928ead6ee2","seconds":587}
{"level":"info","ts":"2026-06-10T10:39:45.349Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"855c8085-93bd-4cf0-8826-858c0de02eb9","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"error","ts":"2026-06-10T10:39:45.352Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"6e7aef73-e7c7-4cf9-9443-d1d9ad307695","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-v8ndd\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:39:45.352Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"6e7aef73-e7c7-4cf9-9443-d1d9ad307695","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-v8ndd\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.352Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"6e7aef73-e7c7-4cf9-9443-d1d9ad307695"}
{"level":"error","ts":"2026-06-10T10:39:45.352Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"6e7aef73-e7c7-4cf9-9443-d1d9ad307695","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-v8ndd\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.354Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"855c8085-93bd-4cf0-8826-858c0de02eb9","name":"kuberay-gateway-access","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:45.357Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"92e6e2f8-474a-498a-8ebf-87c6f6b52c1c","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.357Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"92e6e2f8-474a-498a-8ebf-87c6f6b52c1c","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.359Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"855c8085-93bd-4cf0-8826-858c0de02eb9","name":"test-ns-k92wt-long-running-v8ndd","namespace":"opendatahub"}
{"level":"error","ts":"2026-06-10T10:39:45.359Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"92e6e2f8-474a-498a-8ebf-87c6f6b52c1c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-v8ndd\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:39:45.359Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"92e6e2f8-474a-498a-8ebf-87c6f6b52c1c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-v8ndd\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.359Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"92e6e2f8-474a-498a-8ebf-87c6f6b52c1c"}
{"level":"error","ts":"2026-06-10T10:39:45.359Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"92e6e2f8-474a-498a-8ebf-87c6f6b52c1c","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-v8ndd\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.363Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"855c8085-93bd-4cf0-8826-858c0de02eb9","configMap":"kube-rbac-proxy-config-long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.370Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d6c148a0-ecbe-4871-b89d-3aededf687b2","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.370Z","msg":"Creating self-signed issuer for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d6c148a0-ecbe-4871-b89d-3aededf687b2","rayCluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.370Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"855c8085-93bd-4cf0-8826-858c0de02eb9","serviceAccount":"long-running-v8ndd-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"error","ts":"2026-06-10T10:39:45.373Z","msg":"Failed to create self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d6c148a0-ecbe-4871-b89d-3aededf687b2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-v8ndd\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).createSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:552\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).reconcileSelfSignedIssuer\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:502\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:159\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"error","ts":"2026-06-10T10:39:45.373Z","msg":"Failed to reconcile self-signed issuer","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d6c148a0-ecbe-4871-b89d-3aededf687b2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-v8ndd\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:160\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.373Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d6c148a0-ecbe-4871-b89d-3aededf687b2"}
{"level":"error","ts":"2026-06-10T10:39:45.373Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"d6c148a0-ecbe-4871-b89d-3aededf687b2","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-long-running-v8ndd\" is forbidden: unable to create new content in namespace test-ns-k92wt because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.378Z","logger":"raycluster-resource","msg":"default","name":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.378Z","logger":"raycluster-resource","msg":"enforcing secure trusted network on OpenShift","name":"long-running-v8ndd","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:45.390Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"855c8085-93bd-4cf0-8826-858c0de02eb9","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.390Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"860d84bc-adec-4b1a-868c-72e24cfe7f05","namespacedName":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:45.390Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"860d84bc-adec-4b1a-868c-72e24cfe7f05","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.390Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"860d84bc-adec-4b1a-868c-72e24cfe7f05","cluster":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.390Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"860d84bc-adec-4b1a-868c-72e24cfe7f05","namespace":"test-ns-k92wt","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-06-10T10:39:45.390Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"860d84bc-adec-4b1a-868c-72e24cfe7f05","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-06-10T10:39:45.391Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"562c0cbd-b86f-4506-9a80-4547fad17f31","namespace":"test-ns-k92wt","clusterName":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.391Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"c38dee1d-ed98-4142-aa20-8f340a8e2c78"}
{"level":"info","ts":"2026-06-10T10:39:45.391Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"17d453b2-2f63-4434-8e00-4d590a96a90a","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:45.391Z","logger":"controllers.RayJob","msg":"Failed","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"17d453b2-2f63-4434-8e00-4d590a96a90a","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-06-10 10:39:31 +0000 UTC","Now":"2026-06-10T10:39:45.391Z","ShutdownTime":"2026-06-10T10:49:31.000Z"}
{"level":"info","ts":"2026-06-10T10:39:45.391Z","logger":"controllers.RayJob","msg":"Shutdown behavior is defined by the `ShutdownAfterJobFinishes` flag","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"17d453b2-2f63-4434-8e00-4d590a96a90a","shutdownAfterJobFinishes":true}
{"level":"info","ts":"2026-06-10T10:39:45.391Z","logger":"controllers.RayJob","msg":"shutdownTime not reached, requeue this RayJob for n seconds","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"17d453b2-2f63-4434-8e00-4d590a96a90a","seconds":587}
{"level":"error","ts":"2026-06-10T10:39:45.393Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"860d84bc-adec-4b1a-868c-72e24cfe7f05","error":"rayclusters.ray.io \"long-running-v8ndd\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:214\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.394Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"860d84bc-adec-4b1a-868c-72e24cfe7f05"}
{"level":"error","ts":"2026-06-10T10:39:45.394Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"860d84bc-adec-4b1a-868c-72e24cfe7f05","error":"rayclusters.ray.io \"long-running-v8ndd\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.394Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"5a4380e4-f191-471c-a676-dc79e6901fe7","namespacedName":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:45.394Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"5a4380e4-f191-471c-a676-dc79e6901fe7"}
{"level":"info","ts":"2026-06-10T10:39:45.394Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"5a4380e4-f191-471c-a676-dc79e6901fe7","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:39:45.394Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"5a4380e4-f191-471c-a676-dc79e6901fe7","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:45.395Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"986ff60a-233c-4f84-81d6-df96877f22a3"}
{"level":"info","ts":"2026-06-10T10:39:45.396Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"ac1c1dba-5ad8-49e4-972d-48e533ce738e","namespacedName":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:45.396Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"ac1c1dba-5ad8-49e4-972d-48e533ce738e"}
{"level":"info","ts":"2026-06-10T10:39:45.396Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"ac1c1dba-5ad8-49e4-972d-48e533ce738e","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:39:45.396Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"ac1c1dba-5ad8-49e4-972d-48e533ce738e","namespace":"test-ns-k92wt"}
{"level":"info","ts":"2026-06-10T10:39:45.399Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"bf2ed64c-4a10-4421-b2b7-28b772caaed6","namespacedName":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"}}
{"level":"info","ts":"2026-06-10T10:39:45.399Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"bf2ed64c-4a10-4421-b2b7-28b772caaed6"}
{"level":"info","ts":"2026-06-10T10:39:45.399Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"bf2ed64c-4a10-4421-b2b7-28b772caaed6","namespace":"test-ns-k92wt","authClusters":0}
{"level":"info","ts":"2026-06-10T10:39:45.399Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"bf2ed64c-4a10-4421-b2b7-28b772caaed6","namespace":"test-ns-k92wt"}
{"level":"error","ts":"2026-06-10T10:39:45.405Z","msg":"Failed to delete worker certificate","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"562c0cbd-b86f-4506-9a80-4547fad17f31","certificate":"ray-worker-cert-long-running-v8ndd","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-v8ndd\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:268\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.405Z","logger":"networkpolicy-controller","msg":"RayCluster not found, NetworkPolicies will be garbage collected","controller":"networkpolicy","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"16d1d49d-7496-4f11-9808-93d4746d99c9"}
{"level":"info","ts":"2026-06-10T10:39:45.426Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"562c0cbd-b86f-4506-9a80-4547fad17f31","errorCount":1}
{"level":"info","ts":"2026-06-10T10:39:45.426Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"562c0cbd-b86f-4506-9a80-4547fad17f31"}
{"level":"error","ts":"2026-06-10T10:39:45.426Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"562c0cbd-b86f-4506-9a80-4547fad17f31","error":"certificates.cert-manager.io \"ray-worker-cert-long-running-v8ndd\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.426Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"b83ee4bb-df4a-4d51-b46d-1db38cd59956","namespace":"test-ns-k92wt","clusterName":"long-running-v8ndd"}
{"level":"error","ts":"2026-06-10T10:39:45.430Z","msg":"Failed to delete CA secret","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"b83ee4bb-df4a-4d51-b46d-1db38cd59956","secret":"long-running-v8ndd-ca-secret-e46bc68d","error":"secrets \"long-running-v8ndd-ca-secret-e46bc68d\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).cleanupMTLSResources\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:301\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:144\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.430Z","msg":"Some cleanup operations failed, will retry","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"b83ee4bb-df4a-4d51-b46d-1db38cd59956","errorCount":1}
{"level":"info","ts":"2026-06-10T10:39:45.430Z","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"b83ee4bb-df4a-4d51-b46d-1db38cd59956"}
{"level":"error","ts":"2026-06-10T10:39:45.430Z","msg":"Reconciler error","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"b83ee4bb-df4a-4d51-b46d-1db38cd59956","error":"secrets \"long-running-v8ndd-ca-secret-e46bc68d\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.467Z","msg":"Cleaning up MTLS resources for deleted RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"dfbe41af-0fa4-4da8-b42c-56153d3cd123","namespace":"test-ns-k92wt","clusterName":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.467Z","msg":"Successfully cleaned up MTLS resources","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-v8ndd","namespace":"test-ns-k92wt"},"namespace":"test-ns-k92wt","name":"long-running-v8ndd","reconcileID":"dfbe41af-0fa4-4da8-b42c-56153d3cd123","namespace":"test-ns-k92wt","clusterName":"long-running-v8ndd"}
{"level":"info","ts":"2026-06-10T10:39:45.750Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"8a674122-6217-48cc-a58b-cadfee9fbe46"}
{"level":"info","ts":"2026-06-10T10:39:45.757Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"fa4a5a8e-cd4a-4da5-96a7-87917bcadeca","DeletionTimestamp":"2026-06-10 10:39:45 +0000 UTC"}
{"level":"error","ts":"2026-06-10T10:39:45.757Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"fa4a5a8e-cd4a-4da5-96a7-87917bcadeca","error":"RayCluster.ray.io \"long-running-v8ndd\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.757Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"fa4a5a8e-cd4a-4da5-96a7-87917bcadeca","rayJob":"long-running-pvbmz"}
{"level":"error","ts":"2026-06-10T10:39:45.757Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"fa4a5a8e-cd4a-4da5-96a7-87917bcadeca","error":"Post \"http:///api/jobs/long-running-pvbmz/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:45.757Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"fa4a5a8e-cd4a-4da5-96a7-87917bcadeca","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:39:45.769Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"2b8078a4-634a-4373-8aa9-069b0c2e2c57"}
{"level":"info","ts":"2026-06-10T10:39:46.916Z","msg":"MTLS is enabled for RayCluster","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"c199fc6d-2b90-4021-a881-1fb41f056509","rayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:46.916Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"c199fc6d-2b90-4021-a881-1fb41f056509","pod":"raycluster-head-jjvtb","ip":"10.133.0.13"}
{"level":"info","ts":"2026-06-10T10:39:46.916Z","msg":"Found pod IP","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"c199fc6d-2b90-4021-a881-1fb41f056509","pod":"raycluster-small-group-worker-7j6bp","ip":"10.132.0.41"}
{"level":"info","ts":"2026-06-10T10:39:46.916Z","msg":"All mTLS certificates are ready","controller":"raycluster-mtls","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"},"namespace":"test-ns-ccmjv","name":"raycluster","reconcileID":"c199fc6d-2b90-4021-a881-1fb41f056509","rayCluster":"raycluster","podIPs":["10.133.0.13","10.132.0.41"]}
{"level":"info","ts":"2026-06-10T10:39:48.769Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"long-running","namespace":"test-ns-k92wt"},"reconcileID":"660c7d32-a3bc-4d3e-ab6b-640323a1f958"}
{"level":"info","ts":"2026-06-10T10:39:53.636Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ec024aef-fcf1-4d02-81a9-86a5e33ee98b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.636Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ec024aef-fcf1-4d02-81a9-86a5e33ee98b","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.636Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ec024aef-fcf1-4d02-81a9-86a5e33ee98b","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:39:53.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ec024aef-fcf1-4d02-81a9-86a5e33ee98b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:39:44Z","endTime":"2026-06-10T10:39:48Z"},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:53.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"ec024aef-fcf1-4d02-81a9-86a5e33ee98b","old JobStatus":"","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:39:53.653Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"b1752bc0-833b-4b00-8895-2df1ab85aab9","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.653Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"b1752bc0-833b-4b00-8895-2df1ab85aab9","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.653Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"b1752bc0-833b-4b00-8895-2df1ab85aab9","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:39:53.656Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"b1752bc0-833b-4b00-8895-2df1ab85aab9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:39:44Z","endTime":"2026-06-10T10:39:48Z"},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:39:44Z","endTime":"2026-06-10T10:39:48Z"},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:53.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"46f85632-255c-41e7-9acd-bf1eff499616","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.767Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"46f85632-255c-41e7-9acd-bf1eff499616","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-06-10T10:39:53.776Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"46f85632-255c-41e7-9acd-bf1eff499616"}
{"level":"info","ts":"2026-06-10T10:39:53.776Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"46f85632-255c-41e7-9acd-bf1eff499616","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-06-10T10:39:53.776Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"46f85632-255c-41e7-9acd-bf1eff499616","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-vjx5m","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:39:53.776Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"46f85632-255c-41e7-9acd-bf1eff499616","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-06-10T10:39:53.787Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.787Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.787Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:39:53.787Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","head service name":"raycluster-head-svc","namespace":"test-ns-ccmjv"}
{"level":"info","ts":"2026-06-10T10:39:53.787Z","logger":"controllers.RayJob","msg":"FetchHeadServiceURL","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","head service URL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265"}
{"level":"info","ts":"2026-06-10T10:39:53.787Z","logger":"controllers.RayJob","msg":"user-provided submitter template is used; the first container is assumed to be the submitter","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13"}
{"level":"info","ts":"2026-06-10T10:39:53.787Z","logger":"controllers.RayJob","msg":"No command is specified in the user-provided template. Default command is used","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","command":["if","!","ray","job","status","--address","http://raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","fail-vjx5m",">/dev/null","2>&1",";","then","ray","job","submit","--address","http://raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","--no-wait","--submission-id","fail-vjx5m","--","python /home/ray/jobs/fail.py",";","fi",";","ray","job","logs","--address","http://raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","--follow","fail-vjx5m"]}
{"level":"info","ts":"2026-06-10T10:39:53.802Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-06-10T10:39:53.802Z","logger":"controllers.RayJob","msg":"Both RayCluster and the submitter K8s Job are created. Transition the status from `Initializing` to `Running`.","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","SubmissionMode":"K8sJobMode","RayCluster":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.803Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-vjx5m","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-vjx5m","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-06-10T10:39:53.803Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8fc3e062-b19c-4514-a978-80132769bd13","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:39:53.813Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"c1c064c1-265e-48f2-9b92-518b1ac4ea3f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.813Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"c1c064c1-265e-48f2-9b92-518b1ac4ea3f","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.813Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"c1c064c1-265e-48f2-9b92-518b1ac4ea3f","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:53.816Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"c1c064c1-265e-48f2-9b92-518b1ac4ea3f","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.816Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"c1c064c1-265e-48f2-9b92-518b1ac4ea3f"}
{"level":"error","ts":"2026-06-10T10:39:53.816Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"c1c064c1-265e-48f2-9b92-518b1ac4ea3f","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.822Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"0027a0ce-9ae0-4586-afd3-7766ffb9a8d4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.822Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"0027a0ce-9ae0-4586-afd3-7766ffb9a8d4","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.822Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"0027a0ce-9ae0-4586-afd3-7766ffb9a8d4","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:53.827Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"0027a0ce-9ae0-4586-afd3-7766ffb9a8d4","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.827Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"0027a0ce-9ae0-4586-afd3-7766ffb9a8d4"}
{"level":"error","ts":"2026-06-10T10:39:53.827Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"0027a0ce-9ae0-4586-afd3-7766ffb9a8d4","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.827Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f0aa3bad-db2f-4734-8027-d12ef9584927","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.827Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f0aa3bad-db2f-4734-8027-d12ef9584927","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.827Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f0aa3bad-db2f-4734-8027-d12ef9584927","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:53.831Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f0aa3bad-db2f-4734-8027-d12ef9584927","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.831Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f0aa3bad-db2f-4734-8027-d12ef9584927"}
{"level":"error","ts":"2026-06-10T10:39:53.831Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f0aa3bad-db2f-4734-8027-d12ef9584927","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.838Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"68c21602-e08b-413f-b10c-076b8f274996","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.838Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"68c21602-e08b-413f-b10c-076b8f274996","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.838Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"68c21602-e08b-413f-b10c-076b8f274996","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:53.842Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"68c21602-e08b-413f-b10c-076b8f274996","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.842Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"68c21602-e08b-413f-b10c-076b8f274996"}
{"level":"error","ts":"2026-06-10T10:39:53.842Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"68c21602-e08b-413f-b10c-076b8f274996","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.883Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"cad73678-fbe6-48bb-83e1-764f85fd70c3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.883Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"cad73678-fbe6-48bb-83e1-764f85fd70c3","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.883Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"cad73678-fbe6-48bb-83e1-764f85fd70c3","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:53.902Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"cad73678-fbe6-48bb-83e1-764f85fd70c3","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.902Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"cad73678-fbe6-48bb-83e1-764f85fd70c3"}
{"level":"error","ts":"2026-06-10T10:39:53.902Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"cad73678-fbe6-48bb-83e1-764f85fd70c3","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.983Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"93f747ef-11dd-4d6b-9f21-084161924770","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:53.983Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"93f747ef-11dd-4d6b-9f21-084161924770","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:53.983Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"93f747ef-11dd-4d6b-9f21-084161924770","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:53.988Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"93f747ef-11dd-4d6b-9f21-084161924770","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:53.988Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"93f747ef-11dd-4d6b-9f21-084161924770"}
{"level":"error","ts":"2026-06-10T10:39:53.988Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"93f747ef-11dd-4d6b-9f21-084161924770","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:54.149Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"4f1a29ff-7f49-4384-b105-7b6a4b63a3f0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:54.149Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"4f1a29ff-7f49-4384-b105-7b6a4b63a3f0","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:54.149Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"4f1a29ff-7f49-4384-b105-7b6a4b63a3f0","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:54.153Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"4f1a29ff-7f49-4384-b105-7b6a4b63a3f0","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:54.153Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"4f1a29ff-7f49-4384-b105-7b6a4b63a3f0"}
{"level":"error","ts":"2026-06-10T10:39:54.153Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"4f1a29ff-7f49-4384-b105-7b6a4b63a3f0","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:54.475Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"71c72b03-94b0-46c7-b522-e78589a3a4fb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:54.475Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"71c72b03-94b0-46c7-b522-e78589a3a4fb","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:54.475Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"71c72b03-94b0-46c7-b522-e78589a3a4fb","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:54.479Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"71c72b03-94b0-46c7-b522-e78589a3a4fb","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:54.480Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"71c72b03-94b0-46c7-b522-e78589a3a4fb"}
{"level":"error","ts":"2026-06-10T10:39:54.480Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"71c72b03-94b0-46c7-b522-e78589a3a4fb","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:55.121Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"907bfa2a-3754-40be-8c17-5cd00cdd7e91","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:55.121Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"907bfa2a-3754-40be-8c17-5cd00cdd7e91","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:55.121Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"907bfa2a-3754-40be-8c17-5cd00cdd7e91","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:55.126Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"907bfa2a-3754-40be-8c17-5cd00cdd7e91","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:55.126Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"907bfa2a-3754-40be-8c17-5cd00cdd7e91"}
{"level":"error","ts":"2026-06-10T10:39:55.126Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"907bfa2a-3754-40be-8c17-5cd00cdd7e91","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:55.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"9c95c5ba-4b05-489c-90d1-4f1f7d4b6889","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:55.296Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"9c95c5ba-4b05-489c-90d1-4f1f7d4b6889","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:55.296Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"9c95c5ba-4b05-489c-90d1-4f1f7d4b6889","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:39:55.301Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"9c95c5ba-4b05-489c-90d1-4f1f7d4b6889","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:39:44Z","endTime":"2026-06-10T10:39:48Z"},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:39:44Z","endTime":"2026-06-10T10:39:48Z"},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:56.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"771c84bb-cfa8-4d6f-8481-05fd5b2a7f14","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:56.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"771c84bb-cfa8-4d6f-8481-05fd5b2a7f14","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:56.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"771c84bb-cfa8-4d6f-8481-05fd5b2a7f14","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:56.300Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"771c84bb-cfa8-4d6f-8481-05fd5b2a7f14","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:56.300Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"771c84bb-cfa8-4d6f-8481-05fd5b2a7f14"}
{"level":"error","ts":"2026-06-10T10:39:56.300Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"771c84bb-cfa8-4d6f-8481-05fd5b2a7f14","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:56.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"0a5fa9c0-bfa5-45bb-a97c-dbb23e662674","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:56.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"0a5fa9c0-bfa5-45bb-a97c-dbb23e662674","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:56.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"0a5fa9c0-bfa5-45bb-a97c-dbb23e662674","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:39:56.308Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"0a5fa9c0-bfa5-45bb-a97c-dbb23e662674","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:39:44Z","endTime":"2026-06-10T10:39:48Z"},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:39:44Z","endTime":"2026-06-10T10:39:48Z"},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:56.322Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"17f5429f-3a6a-45d3-ae70-e3242de00dcd","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:56.322Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"17f5429f-3a6a-45d3-ae70-e3242de00dcd","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:56.322Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"17f5429f-3a6a-45d3-ae70-e3242de00dcd","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:39:56.325Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"17f5429f-3a6a-45d3-ae70-e3242de00dcd","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:39:44Z","endTime":"2026-06-10T10:39:48Z"},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-06-10T10:39:35Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:39:44Z","endTime":"2026-06-10T10:39:48Z"},"jobId":"counter-9wf7b","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-06-10T10:39:35Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:39:56.325Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"17f5429f-3a6a-45d3-ae70-e3242de00dcd","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-06-10T10:39:56.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"89ef2ac0-3a98-402f-9318-490fa78b1a9f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:56.337Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"89ef2ac0-3a98-402f-9318-490fa78b1a9f","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:39:56 +0000 UTC","Now":"2026-06-10T10:39:56.337Z","ShutdownTime":"2026-06-10T10:39:56.000Z"}
{"level":"info","ts":"2026-06-10T10:39:56.407Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"3249ae7b-51f6-4a3a-b3fd-09d84093c8fe","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:56.407Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"3249ae7b-51f6-4a3a-b3fd-09d84093c8fe","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:39:56.407Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"3249ae7b-51f6-4a3a-b3fd-09d84093c8fe","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"error","ts":"2026-06-10T10:39:56.409Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"3249ae7b-51f6-4a3a-b3fd-09d84093c8fe","JobId":"fail-vjx5m","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:282\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:56.409Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"3249ae7b-51f6-4a3a-b3fd-09d84093c8fe"}
{"level":"error","ts":"2026-06-10T10:39:56.409Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"3249ae7b-51f6-4a3a-b3fd-09d84093c8fe","error":"Job fail-vjx5m does not exist on the cluster","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-06-10T10:39:56.654Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"95560b7d-b100-4843-abcc-2e231f74faad","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:39:56.654Z","logger":"controllers.RayJob","msg":"Complete","RayJob":{"name":"counter","namespace":"test-ns-ccmjv"},"reconcileID":"95560b7d-b100-4843-abcc-2e231f74faad","ShutdownAfterJobFinishes":false,"ClusterSelector":{"ray.io/cluster":"raycluster"},"ttlSecondsAfterFinished":0,"Status.endTime":"2026-06-10 10:39:56 +0000 UTC","Now":"2026-06-10T10:39:56.654Z","ShutdownTime":"2026-06-10T10:39:56.000Z"}
{"level":"info","ts":"2026-06-10T10:40:01.443Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-k92wt"},"reconcileID":"d1c2002e-8024-4cd7-a531-7b3ab818f12d"}
{"level":"info","ts":"2026-06-10T10:40:01.530Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"7d00e883-3967-4f86-ba62-b9bcf555f608","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:40:01.530Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"7d00e883-3967-4f86-ba62-b9bcf555f608","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:40:01.530Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"7d00e883-3967-4f86-ba62-b9bcf555f608","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:40:01.535Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"7d00e883-3967-4f86-ba62-b9bcf555f608","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-vjx5m","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:40:00Z"},"jobId":"fail-vjx5m","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:40:01.535Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"7d00e883-3967-4f86-ba62-b9bcf555f608","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:40:01.548Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f8dbed85-8b22-4833-a10d-a0fcb42499b5","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:40:01.548Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f8dbed85-8b22-4833-a10d-a0fcb42499b5","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:40:01.548Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f8dbed85-8b22-4833-a10d-a0fcb42499b5","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:40:01.550Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"f8dbed85-8b22-4833-a10d-a0fcb42499b5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:40:00Z"},"jobId":"fail-vjx5m","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:40:00Z"},"jobId":"fail-vjx5m","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:40:04.549Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"ea13cf56-e8d1-4a45-973a-1a213a13f52c","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:40:04.549Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"ea13cf56-e8d1-4a45-973a-1a213a13f52c","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:40:04.549Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"ea13cf56-e8d1-4a45-973a-1a213a13f52c","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:40:04.552Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"ea13cf56-e8d1-4a45-973a-1a213a13f52c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:40:00Z"},"jobId":"fail-vjx5m","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:40:00Z","endTime":"2026-06-10T10:40:02Z"},"jobId":"fail-vjx5m","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:40:00,793\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-vjx5m: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-06-10T10:40:04.552Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"ea13cf56-e8d1-4a45-973a-1a213a13f52c","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-06-10T10:40:04.567Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8d8570bb-b369-4e32-ae9f-1e93ab62e70f","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-06-10T10:40:04.567Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8d8570bb-b369-4e32-ae9f-1e93ab62e70f","name":"raycluster"}
{"level":"info","ts":"2026-06-10T10:40:04.567Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8d8570bb-b369-4e32-ae9f-1e93ab62e70f","RayCluster":{"name":"raycluster","namespace":"test-ns-ccmjv"}}
{"level":"info","ts":"2026-06-10T10:40:04.570Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-ccmjv"},"reconcileID":"8d8570bb-b369-4e32-ae9f-1e93ab62e70f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:40:00Z","endTime":"2026-06-10T10:40:02Z"},"jobId":"fail-vjx5m","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:40:00,793\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-vjx5m: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-06-10T10:40:00Z","endTime":"2026-06-10T10:40:02Z"},"jobId":"fail-vjx5m","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-ccmjv.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-06-10 10:40:00,793\tINFO job_manager.py:587 -- Runtime env is setting up.\nRunning entrypoint for job fail-vjx5m: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-06-10T10:39:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-06-10T10:28:51Z","stateTransitionTimes":{"ready":"2026-06-10T10:28:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.133.0.13","serviceIP":"10.133.0.13","podName":"raycluster-head-jjvtb","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-06-10T10:25:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-06-10T10:28:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-06-10T10:24:16Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-06-10T10:24:16Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}